level=info ts=2026-04-21T10:18:37.852337958Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T10:18:37.852375975Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T10:18:37.854457513Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T10:18:37.854900177Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T10:18:37.855002271Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.62 inf=eth0
level=debug ts=2026-04-21T10:18:37.855679932Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T10:18:37.855766579Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.62 inf=eth0
level=info ts=2026-04-21T10:18:37.855774595Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-8dac3323
ts=2026-04-21T10:18:37.855847914Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T10:18:37.855863558Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.62 advertisePort=7946
level=debug ts=2026-04-21T10:18:37.855884511Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.62 advertisePort=7946
level=info ts=2026-04-21T10:18:37.856034902Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T10:18:37.860829777Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T10:18:37.860870906Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T10:18:37.860884859Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.852429ms
level=error ts=2026-04-21T10:18:37.860894664Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T10:18:37.860909371Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T10:18:37.86328284Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:18:37.863297469Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T10:18:37.869982371Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T10:18:37.869991005Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T10:18:37.869998982Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T10:18:37.870017253Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T10:18:37.870015832Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T10:18:37.870041575Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T10:18:37.870086386Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T10:18:37.870112739Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T10:18:37.870372646Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T10:18:37.870383654Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T10:18:37.870435264Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T10:18:37.870449562Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T10:18:37.870466815Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T10:18:37.870610453Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T10:18:37.870617414Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T10:18:37.870627367Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T10:18:38.871267296Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T10:18:38.871294313Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T10:18:38.871318129Z caller=loki.go:599 msg="Loki started" startup_time=1.086776662s
ts=2026-04-21T10:18:39.574621507Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:18:39.574641398Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:18:43.232045113Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:18:43.232073475Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T10:18:43.872158759Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T10:18:43.872225382Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T10:18:49.47608341Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:18:49.476110972Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T10:18:56.758685641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:18:57.502070788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37802"
ts=2026-04-21T10:18:58.563368997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52792"
ts=2026-04-21T10:19:01.200849362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.62:7946"
ts=2026-04-21T10:19:01.200954799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:55918"
ts=2026-04-21T10:19:01.201526915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.61:7946"
ts=2026-04-21T10:19:01.202970552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.78:7946"
level=info ts=2026-04-21T10:19:01.204217428Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.343300813s
ts=2026-04-21T10:19:02.621657235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49614"
level=debug ts=2026-04-21T10:19:14.324212037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:19:14.359937924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49660"
ts=2026-04-21T10:19:16.322911381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:19:38.451569721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:19:46.323918721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:19:58.814771337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54704"
ts=2026-04-21T10:19:59.489968962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:55582"
level=debug ts=2026-04-21T10:20:00.301406132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:20:16.325021905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:20:27.093108172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T10:20:27.094998036Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:20:28.816741323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45828"
ts=2026-04-21T10:20:29.491340932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:49626"
ts=2026-04-21T10:20:46.326560721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:21:11.251300173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:21:16.327932082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:21:29.495302643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:51892"
ts=2026-04-21T10:21:46.32960919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:21:59.49635564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:57978"
level=debug ts=2026-04-21T10:22:14.52925956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:22:16.332336717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:22:44.381226959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41478"
ts=2026-04-21T10:22:46.334917429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:23:14.383984137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35730"
ts=2026-04-21T10:23:16.337931862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:23:29.502374017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:48464"
level=debug ts=2026-04-21T10:23:33.127407522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:23:46.339836046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:24:14.389728833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49248"
ts=2026-04-21T10:24:16.342077518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:24:44.393063994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51852"
ts=2026-04-21T10:24:46.343181909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:24:46.497087462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:25:16.344595656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:25:46.345824224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:25:57.348974118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T10:25:57.350498718Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:26:16.347815899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:26:28.84818967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49940"
ts=2026-04-21T10:26:44.404077503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52044"
ts=2026-04-21T10:26:46.35013803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:26:49.785985721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:27:16.353325007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:27:28.853918962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59178"
level=debug ts=2026-04-21T10:27:31.355945276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:27:46.355579532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:27:59.526533259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:49616"
level=debug ts=2026-04-21T10:28:07.401053448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:28:14.413272088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46318"
ts=2026-04-21T10:28:16.357974201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=info ts=2026-04-21T10:28:43.873220756Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T10:28:43.876916142Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:28:43.878739738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:28:43.880340628Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:28:43.880367481Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T10:28:43.880379372Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T10:28:44.416637854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49036"
ts=2026-04-21T10:28:46.360390094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:28:58.861068168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59456"
level=debug ts=2026-04-21T10:29:01.516098487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:29:14.41987963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35482"
ts=2026-04-21T10:29:16.361876903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:29:29.534492664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:52458"
ts=2026-04-21T10:29:46.364545647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:29:49.355308472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:29:49.35726332Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:29:58.864796175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56216"
ts=2026-04-21T10:30:16.367609965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:30:28.866649801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55154"
ts=2026-04-21T10:30:29.53809503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50286"
level=debug ts=2026-04-21T10:30:41.301399727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:30:44.429340113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52128"
ts=2026-04-21T10:30:46.369377779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:31:16.371896789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:31:25.603718683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:31:29.541621711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50408"
ts=2026-04-21T10:31:44.435047941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36610"
ts=2026-04-21T10:31:46.374561266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:32:12.862215345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:32:16.37688361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:32:44.441428178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58644"
ts=2026-04-21T10:32:46.378992175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:33:05.460134808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:33:16.38066736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:33:46.382532069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:33:49.703554507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:33:49.705361769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:33:58.88319227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:51774"
ts=2026-04-21T10:34:14.451110785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51808"
ts=2026-04-21T10:34:16.384998004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:34:46.38751145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:34:47.775868671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:34:59.558990499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:46214"
ts=2026-04-21T10:35:16.389128038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:35:27.386510688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:35:44.461376025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:55508"
ts=2026-04-21T10:35:46.390872423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:35:59.562333846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:49554"
ts=2026-04-21T10:36:16.393943468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:36:25.780359904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:36:28.8954188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41294"
ts=2026-04-21T10:36:29.563593684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:34106"
ts=2026-04-21T10:36:46.396261177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:36:59.565175041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54520"
level=debug ts=2026-04-21T10:37:02.502813526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:37:14.470176809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43526"
ts=2026-04-21T10:37:16.399367922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:37:29.566717047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:38540"
level=debug ts=2026-04-21T10:37:40.2554577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:37:40.25709756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:37:46.40243776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:38:15.956313476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:38:16.404629692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:38:28.906650821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38396"
level=error ts=2026-04-21T10:38:43.887680074Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:38:43.889572559Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:38:43.891269533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:38:43.891297984Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:38:46.406840682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:38:58.909033592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54782"
level=debug ts=2026-04-21T10:39:09.737384696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:39:16.408782425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:39:28.911551345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52662"
ts=2026-04-21T10:39:29.576664216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:48054"
ts=2026-04-21T10:39:46.411193039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:39:50.083806234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:39:58.913616678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:44068"
ts=2026-04-21T10:40:14.49014986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:57794"
ts=2026-04-21T10:40:16.41401807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:40:28.915346876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56136"
level=debug ts=2026-04-21T10:40:32.636906535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:40:46.416141694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:40:58.91809398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:39528"
ts=2026-04-21T10:41:16.418137576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:41:29.160745249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:41:29.162345836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:41:29.584904775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:56576"
ts=2026-04-21T10:41:46.419651109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:41:58.922463597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52112"
level=debug ts=2026-04-21T10:42:03.639396725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:42:16.421157471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:42:42.50540959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:42:46.423830528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:42:58.926782407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33960"
ts=2026-04-21T10:43:14.510509922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54312"
ts=2026-04-21T10:43:16.425690751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:43:28.928643371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37160"
level=debug ts=2026-04-21T10:43:29.160925019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:43:46.42701079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:43:58.930949233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56514"
ts=2026-04-21T10:44:16.428796568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:44:25.007735474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:44:29.598505467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:43718"
ts=2026-04-21T10:44:46.431981827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:45:09.80447722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:45:09.806433013Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:45:16.434404098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:45:46.437572809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:45:50.513869035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:45:58.942361286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55012"
ts=2026-04-21T10:46:14.530043093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36912"
ts=2026-04-21T10:46:16.440710589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:46:42.642328171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:46:46.442741497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:46:58.947797159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:40764"
ts=2026-04-21T10:46:59.61150584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:56678"
ts=2026-04-21T10:47:16.445349411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:47:22.121480199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:47:29.612915405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39766"
ts=2026-04-21T10:47:44.543005291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54170"
ts=2026-04-21T10:47:46.448745954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:47:59.614381209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39410"
level=debug ts=2026-04-21T10:48:12.856379518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:48:14.546508534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47746"
ts=2026-04-21T10:48:16.452204824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:48:29.615830696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:36944"
level=error ts=2026-04-21T10:48:43.885942461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:48:43.887795848Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:48:43.889479287Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:48:43.88950673Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:48:46.454863294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:49:12.344619827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:49:12.346488842Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:49:16.457207231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:49:29.619365047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45244"
ts=2026-04-21T10:49:44.556642028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48482"
ts=2026-04-21T10:49:46.459400027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:50:08.433796388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:50:16.461268297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:50:44.633692396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:50:46.463869639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:51:14.56614324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56584"
ts=2026-04-21T10:51:16.466048555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:51:25.949496696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:51:46.467979016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:51:58.973922467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48806"
ts=2026-04-21T10:52:16.469690851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:52:21.702026558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:52:28.976284712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55518"
ts=2026-04-21T10:52:44.575669476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47074"
ts=2026-04-21T10:52:46.47157046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:53:05.328513544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:53:05.330075136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:53:16.473275582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T10:53:43.035691791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:53:46.4744926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:53:59.642782288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:56024"
ts=2026-04-21T10:54:16.476068206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:54:23.004354754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:54:44.58757289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49376"
ts=2026-04-21T10:54:46.477463219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:55:16.480120599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T10:55:24.934794223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:55:44.593877427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58626"
ts=2026-04-21T10:55:46.481780076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:55:58.993562404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58674"
ts=2026-04-21T10:55:59.653841624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:34466"
level=debug ts=2026-04-21T10:56:03.380680708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:56:14.597073662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46900"
ts=2026-04-21T10:56:16.483619557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:56:28.997129399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:60578"
level=debug ts=2026-04-21T10:56:39.460129146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:56:39.461817349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:56:46.486401511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T10:57:16.488754537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:57:29.660321572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:33922"
level=debug ts=2026-04-21T10:57:29.738106911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:57:44.606580186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56608"
ts=2026-04-21T10:57:46.491034059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T10:57:59.005369894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45224"
ts=2026-04-21T10:57:59.661980479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:38604"
ts=2026-04-21T10:58:16.492801891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:58:29.663558086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:58156"
level=debug ts=2026-04-21T10:58:30.158138115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T10:58:43.883656152Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:58:43.885537258Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:58:43.887466104Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T10:58:43.887495839Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T10:58:44.612569045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43304"
ts=2026-04-21T10:58:46.494417987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T10:59:14.615412916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46916"
ts=2026-04-21T10:59:16.495386432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T10:59:21.875634099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T10:59:46.49726714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
level=debug ts=2026-04-21T11:00:00.581547944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:00:16.499121312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T11:00:29.018337158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48672"
level=debug ts=2026-04-21T11:00:43.575139385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:00:43.576739588Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:00:44.62554429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:60758"
ts=2026-04-21T11:00:46.502561861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T11:01:16.5060657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T11:01:29.024074662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56556"
level=debug ts=2026-04-21T11:01:39.625971243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:01:46.508020971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T11:01:59.025992943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:40748"
ts=2026-04-21T11:01:59.681496765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54364"
ts=2026-04-21T11:02:14.635324348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47312"
level=debug ts=2026-04-21T11:02:15.125449527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:02:16.510230328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T11:02:29.682346101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:38096"
ts=2026-04-21T11:02:46.512972283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T11:02:50.602023647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:03:16.514494127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T11:03:37.131762065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:03:46.516093971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
ts=2026-04-21T11:03:59.689542872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:34236"
ts=2026-04-21T11:04:16.518204774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tpq7c-cb588ed6 10.129.0.61:7946"
level=debug ts=2026-04-21T11:04:37.069830612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:04:37.071470197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:04:46.520397597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
ts=2026-04-21T11:04:59.693500773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:38418"
ts=2026-04-21T11:05:16.522424621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7935d0e2 10.128.0.67:7946"
level=debug ts=2026-04-21T11:05:39.093635905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:05:44.656748564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53150"
ts=2026-04-21T11:05:46.524928879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2c2gg-7bf53573 10.130.0.78:7946"
ts=2026-04-21T11:05:59.697296617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45504"