level=info ts=2026-04-21T19:50:48.962802413Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T19:50:48.962844449Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T19:50:48.965405676Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T19:50:48.965869203Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T19:50:48.965993423Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.50 inf=eth0
level=debug ts=2026-04-21T19:50:48.966632411Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T19:50:48.966760253Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.50 inf=eth0
level=info ts=2026-04-21T19:50:48.966792196Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-850c7384
ts=2026-04-21T19:50:48.966865756Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T19:50:48.966878194Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.50 advertisePort=7946
level=debug ts=2026-04-21T19:50:48.966892993Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.50 advertisePort=7946
level=info ts=2026-04-21T19:50:48.966998874Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T19:50:48.974677179Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T19:50:48.9747Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T19:50:48.974710295Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.714017ms
level=error ts=2026-04-21T19:50:48.974717395Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T19:50:48.974726009Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T19:50:48.977576045Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T19:50:48.977595391Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T19:50:48.983463883Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T19:50:48.983513362Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-21T19:50:48.983532145Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T19:50:48.98354091Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-21T19:50:48.983531918Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T19:50:48.983609712Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T19:50:48.983633929Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T19:50:48.983640825Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T19:50:48.983810158Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T19:50:48.983822412Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T19:50:48.983884942Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T19:50:48.983872076Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T19:50:48.983899222Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T19:50:48.984022697Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T19:50:49.153715475Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=info ts=2026-04-21T19:50:49.153871485Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T19:50:49.277940299Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T19:50:49.278001776Z caller=loki.go:599 msg="Loki started" startup_time=377.779177ms
ts=2026-04-21T19:50:50.773596145Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T19:50:50.773621702Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T19:50:54.278294197Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T19:50:54.278361299Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T19:50:54.57935318Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T19:50:54.579395905Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:50:59.412745673Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T19:50:59.412771622Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T19:51:03.867942352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:51:09.131091684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:53992"
ts=2026-04-21T19:51:12.067856812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:49146"
ts=2026-04-21T19:51:13.921518506Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.49:7946: dial tcp 10.130.0.49:7946: connect: connection refused"
ts=2026-04-21T19:51:13.921636433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.50:7946"
ts=2026-04-21T19:51:13.921662269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33596"
ts=2026-04-21T19:51:13.92367903Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.52:7946: dial tcp 10.129.0.52:7946: connect: connection refused"
ts=2026-04-21T19:51:13.924227211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.49:7946"
ts=2026-04-21T19:51:13.925498722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.51:7946"
ts=2026-04-21T19:51:13.927232992Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.50:7946: dial tcp 10.129.0.50:7946: connect: connection refused"
level=info ts=2026-04-21T19:51:13.927243107Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.952512722s
ts=2026-04-21T19:51:13.996127781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:46936"
level=debug ts=2026-04-21T19:51:20.79205934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:51:23.902755216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:49608"
ts=2026-04-21T19:51:36.929067202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:51886"
ts=2026-04-21T19:51:38.141490829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:51:39.546620153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T19:52:06.58502293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:52:08.143721443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T19:52:14.028092022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40396"
ts=2026-04-21T19:52:38.146127601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:52:38.766309199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T19:52:38.76826407Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:53:08.148189749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:53:26.695025308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:53:38.150014746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T19:54:08.152446901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T19:54:38.154659088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:54:39.078725571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:54:53.912983774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:35582"
ts=2026-04-21T19:55:08.156562977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:55:34.893630529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:55:38.158745543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T19:55:53.91644387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:37534"
ts=2026-04-21T19:56:08.161089008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T19:56:38.16298565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:56:47.541127568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T19:57:08.165508198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T19:57:14.043821328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:34642"
ts=2026-04-21T19:57:38.167866377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T19:57:44.045914027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:33052"
level=debug ts=2026-04-21T19:57:59.065777999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T19:57:59.067564627Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:58:06.946210083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:46640"
ts=2026-04-21T19:58:08.170012307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T19:58:14.047624688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36352"
ts=2026-04-21T19:58:38.172491335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T19:58:44.049528225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54806"
ts=2026-04-21T19:58:53.925064216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:35686"
level=debug ts=2026-04-21T19:58:55.005517344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:59:08.174670791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T19:59:14.051638329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41698"
ts=2026-04-21T19:59:36.951977944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:38216"
ts=2026-04-21T19:59:38.176691747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T19:59:51.709569494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:00:08.178467595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:00:31.164317822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:00:36.95584488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:42966"
ts=2026-04-21T20:00:38.181243671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=info ts=2026-04-21T20:00:54.27868648Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T20:00:54.283960543Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:00:54.285841242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:00:54.287692673Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:00:54.287721548Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T20:00:54.287733862Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T20:01:08.183315823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:01:23.932843439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:36894"
level=debug ts=2026-04-21T20:01:29.347332259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:01:38.185180905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:01:53.934901644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:37358"
level=debug ts=2026-04-21T20:02:04.234593751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:02:04.236237508Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:02:08.187091002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:02:36.961696952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:57716"
ts=2026-04-21T20:02:38.188959474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:02:41.259313563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:02:53.937837578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:34542"
ts=2026-04-21T20:03:06.963763386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:59508"
ts=2026-04-21T20:03:08.191559718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:03:23.93957605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:40300"
level=debug ts=2026-04-21T20:03:32.96845403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:03:36.965794139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:56388"
ts=2026-04-21T20:03:38.194424926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:03:53.941622137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:46880"
ts=2026-04-21T20:04:08.196867839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:04:14.067360141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:46936"
ts=2026-04-21T20:04:23.943602806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:49896"
level=debug ts=2026-04-21T20:04:33.107197495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:04:36.969580853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:60210"
ts=2026-04-21T20:04:38.199122009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:05:06.971825008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:40396"
ts=2026-04-21T20:05:08.2013121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:05:23.537940807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:05:23.947670828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:34576"
ts=2026-04-21T20:05:38.203817689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:05:44.07270986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47608"
ts=2026-04-21T20:05:53.950062693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:46478"
level=debug ts=2026-04-21T20:05:59.181558014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:05:59.183288426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:06:08.205869531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:06:38.208543763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:06:44.075838083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58092"
level=debug ts=2026-04-21T20:06:47.002944857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:06:53.95309996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:53628"
ts=2026-04-21T20:07:06.977994768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:33258"
ts=2026-04-21T20:07:08.211184943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:07:36.980110474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:42802"
ts=2026-04-21T20:07:38.213014296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:07:38.918003438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:07:44.078952864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:46674"
ts=2026-04-21T20:07:53.956787859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:55436"
ts=2026-04-21T20:08:08.214867593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:08:14.081382047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:34014"
level=debug ts=2026-04-21T20:08:17.001834456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:08:38.216963391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:08:51.742752031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:09:08.219710584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:09:32.244602638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:09:32.246168558Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:09:38.221817302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:09:53.963083628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:51242"
ts=2026-04-21T20:10:08.223614793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:10:14.087899112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:56082"
level=debug ts=2026-04-21T20:10:24.327612632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:10:36.988348579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:42588"
ts=2026-04-21T20:10:38.225903577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:10:53.966652045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:48246"
level=error ts=2026-04-21T20:10:54.293717235Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:10:54.296062408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:10:54.298064013Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:10:54.29809988Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:11:08.228716167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:11:14.090860102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48058"
level=debug ts=2026-04-21T20:11:20.453218108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:11:36.991921621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:51096"
ts=2026-04-21T20:11:38.230354895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:11:53.969917081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:36704"
ts=2026-04-21T20:12:08.232581986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:12:13.647559184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:12:14.093831489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:53966"
ts=2026-04-21T20:12:38.234813925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:12:44.096029611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:44058"
level=debug ts=2026-04-21T20:12:48.118648312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:13:08.23691539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:13:23.975130931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:46434"
ts=2026-04-21T20:13:36.998416447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:39412"
ts=2026-04-21T20:13:38.23899842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:13:44.099441269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:60070"
level=debug ts=2026-04-21T20:13:46.4870349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:13:46.488708669Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:14:08.241238066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:14:14.101315604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:60580"
ts=2026-04-21T20:14:23.978958969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:45778"
level=debug ts=2026-04-21T20:14:24.451618185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:14:38.243727478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:14:44.103638774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36298"
ts=2026-04-21T20:15:08.245393774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:15:22.16906932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:15:37.005878505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:38644"
ts=2026-04-21T20:15:38.248019477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:16:08.250617552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:16:23.984598119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:34486"
level=debug ts=2026-04-21T20:16:24.144117821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:16:38.253034139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:16:53.986779249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:60626"
ts=2026-04-21T20:17:07.010246167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:55328"
ts=2026-04-21T20:17:08.255571407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:17:16.182922149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:17:38.257898544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:18:07.922233317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:18:07.924299245Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:18:08.261820296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:18:38.267155967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:18:44.113788076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:38592"
level=debug ts=2026-04-21T20:18:49.643175962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:18:53.99333195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:50186"
ts=2026-04-21T20:19:08.272691673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:19:37.017146582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:52694"
ts=2026-04-21T20:19:38.275214913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:19:44.117132674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58856"
level=debug ts=2026-04-21T20:19:45.949039937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:20:08.277270576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:20:21.024281626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:20:23.998276104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:55196"
ts=2026-04-21T20:20:38.279463244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:20:44.120550823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:39066"
ts=2026-04-21T20:20:54.000670137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:38576"
level=error ts=2026-04-21T20:20:54.290520932Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:20:54.292274721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:20:54.294057379Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:20:54.294090119Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T20:20:56.084097056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:21:07.022437736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:42798"
ts=2026-04-21T20:21:08.281872664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:21:31.486937025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:21:31.488526501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:21:37.024262014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:41776"
ts=2026-04-21T20:21:38.284061064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:21:54.004397867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:33534"
ts=2026-04-21T20:22:07.026072334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:50408"
ts=2026-04-21T20:22:08.285930141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:22:20.07650849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:22:38.288541426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:23:01.591313357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:23:08.290365686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:23:14.128957702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:52266"
ts=2026-04-21T20:23:38.2921226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:24:01.852182065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:24:08.294243373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:24:24.012059025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:46024"
ts=2026-04-21T20:24:38.296151494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:24:46.947921472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:25:07.036038161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:41252"
ts=2026-04-21T20:25:08.298513303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:25:32.015793607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:25:32.017349919Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:25:38.300954656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:25:54.016825129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:48428"
ts=2026-04-21T20:26:08.303517086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:26:23.247522786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:26:37.041525423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:37340"
ts=2026-04-21T20:26:38.306290616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:26:44.139892257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40882"
ts=2026-04-21T20:26:54.020607451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:40990"
level=debug ts=2026-04-21T20:27:01.130706297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:27:08.308980986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:27:37.045377438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:50782"
ts=2026-04-21T20:27:38.311605213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:27:44.143920429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51846"
level=debug ts=2026-04-21T20:27:54.304998643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:28:08.313462323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:28:14.1466663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55770"
level=debug ts=2026-04-21T20:28:30.032766612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:28:38.315631355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:29:04.619913474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:29:04.621935075Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:29:08.318280137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:29:14.150544215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36558"
ts=2026-04-21T20:29:24.028516928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:48388"
ts=2026-04-21T20:29:37.052345328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:34452"
ts=2026-04-21T20:29:38.321052016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:29:44.153099419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51296"
ts=2026-04-21T20:29:54.035626461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:46300"
level=debug ts=2026-04-21T20:29:54.283350827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:30:07.054446693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:47572"
ts=2026-04-21T20:30:08.323712461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:30:14.154817685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:33164"
level=debug ts=2026-04-21T20:30:36.207886487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:30:38.327410406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:30:44.157647952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:39872"
ts=2026-04-21T20:30:54.045733745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:41948"
level=error ts=2026-04-21T20:30:54.291537567Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:30:54.293334474Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:30:54.295407425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:30:54.295435154Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:31:08.331574284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:31:37.003577988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:31:37.059450713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:60110"
ts=2026-04-21T20:31:38.33777465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:32:08.341709775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:32:24.050696239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:34606"
level=debug ts=2026-04-21T20:32:27.819794354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:32:37.062613105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:39498"
ts=2026-04-21T20:32:38.346382513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:32:44.163436942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59398"
ts=2026-04-21T20:32:54.052739423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:58156"
level=debug ts=2026-04-21T20:33:07.349922123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:33:07.351754537Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:33:08.348251528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:33:38.350331954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:34:07.067511843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:39870"
level=debug ts=2026-04-21T20:34:07.263141206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:34:08.352303754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:34:38.354786957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:34:54.059361086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:59088"
level=debug ts=2026-04-21T20:35:05.767779164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:35:07.070535367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:51858"
ts=2026-04-21T20:35:08.356651973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:35:14.170510556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:52112"
ts=2026-04-21T20:35:24.061216757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:53658"
ts=2026-04-21T20:35:37.07370515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:40072"
ts=2026-04-21T20:35:38.362668911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:35:53.349119241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:36:08.368090066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:36:33.621433671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:36:38.370807415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:37:07.078763259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:58642"
ts=2026-04-21T20:37:08.3725868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:37:16.044797814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:37:16.04677438Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:37:24.067053082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:37736"
ts=2026-04-21T20:37:38.374419501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:37:54.069546956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:35648"
ts=2026-04-21T20:38:08.37731555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:38:13.326797213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:38:14.180271098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:44548"
ts=2026-04-21T20:38:38.379491422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:38:44.182140834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54302"
ts=2026-04-21T20:38:54.073222872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:59472"
ts=2026-04-21T20:39:08.381116563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:39:09.61395762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:39:37.087201017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:54590"
ts=2026-04-21T20:39:38.382976217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:39:48.661271055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:40:08.384784138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:40:35.472203412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:40:38.387306183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:40:54.078211139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:59760"
level=error ts=2026-04-21T20:40:54.29172586Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:40:54.29365832Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:40:54.295348368Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T20:40:54.295384578Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:41:07.091421628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:46398"
ts=2026-04-21T20:41:08.389655055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:41:14.189124587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36896"
level=debug ts=2026-04-21T20:41:30.562385003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:41:30.564444192Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:41:37.093086095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:49738"
ts=2026-04-21T20:41:38.392286182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:41:54.080906586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:50596"
ts=2026-04-21T20:42:07.095122471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:39206"
ts=2026-04-21T20:42:08.394194916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:42:09.171033855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:42:37.09711333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:34672"
ts=2026-04-21T20:42:38.39584422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:42:54.08463076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:52188"
level=debug ts=2026-04-21T20:43:00.753853945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:43:08.398338407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:43:38.400953378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:43:41.880133183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:44:07.100237136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:50924"
ts=2026-04-21T20:44:08.403207328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:44:14.19854469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47930"
level=debug ts=2026-04-21T20:44:27.641580721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:44:38.405660212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
ts=2026-04-21T20:45:08.408007341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:45:17.413531964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:45:17.415667721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:45:38.410289968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:45:44.202721119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:45402"
level=debug ts=2026-04-21T20:45:55.354172299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:46:08.412394266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsbg4-2ac3d68c 10.129.0.51:7946"
level=debug ts=2026-04-21T20:46:33.691731603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:46:37.107439178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:60634"
ts=2026-04-21T20:46:38.414724545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
ts=2026-04-21T20:46:44.206156861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:57380"
ts=2026-04-21T20:46:54.095783121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:40704"
ts=2026-04-21T20:47:07.109448948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:45708"
ts=2026-04-21T20:47:08.41704633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:47:24.09748408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.51:45082"
level=debug ts=2026-04-21T20:47:24.289459903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:47:37.111147451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:40098"
ts=2026-04-21T20:47:38.422518963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
level=debug ts=2026-04-21T20:47:58.902448549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T20:48:07.113702844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.49:39246"
ts=2026-04-21T20:48:08.424877672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"
ts=2026-04-21T20:48:14.210045345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58552"
ts=2026-04-21T20:48:38.426819039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fed1b16f 10.129.0.53:7946"
level=debug ts=2026-04-21T20:48:58.390866936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T20:48:58.392435828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T20:49:08.428340262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9slwt-94e35a9c 10.129.0.49:7946"