level=info ts=2026-04-20T18:56:48.243154705Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T18:56:48.243192864Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T18:56:48.244418922Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T18:56:48.244906787Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T18:56:48.245024153Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=debug ts=2026-04-20T18:56:48.24644236Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T18:56:48.246538325Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=info ts=2026-04-20T18:56:48.246640571Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-3cac7ad0
ts=2026-04-20T18:56:48.246713742Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T18:56:48.246732958Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=debug ts=2026-04-20T18:56:48.246749956Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=info ts=2026-04-20T18:56:48.246894878Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T18:56:48.253746638Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T18:56:48.253769501Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T18:56:48.25378542Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.893541ms
level=error ts=2026-04-20T18:56:48.253792777Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T18:56:48.253805897Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T18:56:48.256348981Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T18:56:48.256380683Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T18:56:48.264682558Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-20T18:56:48.264690041Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T18:56:48.264715116Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T18:56:48.264703782Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-20T18:56:48.264724287Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-20T18:56:48.264713588Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T18:56:48.265042295Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-20T18:56:48.265052853Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T18:56:48.265057225Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-20T18:56:48.26506107Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T18:56:48.265114985Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T18:56:48.265114507Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T18:56:48.26514383Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T18:56:48.265296627Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T18:56:48.265308158Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T18:56:48.265328837Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T18:56:49.266432337Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T18:56:49.374665856Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T18:56:49.374726749Z caller=loki.go:599 msg="Loki started" startup_time=1.215318055s
ts=2026-04-20T18:56:49.490361296Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T18:56:49.490388018Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:56:51.708356996Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T18:56:51.70838208Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T18:56:54.37476329Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T18:56:54.374853844Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T18:56:56.069435871Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T18:56:56.069460981Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T18:57:02.01700247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:57:05.048498149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.65:7946"
ts=2026-04-20T18:57:05.048605278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50428"
ts=2026-04-20T18:57:05.050118948Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.64:7946: dial tcp 10.130.0.64:7946: connect: connection refused"
ts=2026-04-20T18:57:05.052268158Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.100:7946: dial tcp 10.129.0.100:7946: connect: connection refused"
ts=2026-04-20T18:57:05.053860339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.98:7946"
level=info ts=2026-04-20T18:57:05.055565557Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=16.801755054s
ts=2026-04-20T18:57:11.562813746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:53878"
ts=2026-04-20T18:57:13.3319684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:44852"
ts=2026-04-20T18:57:18.363539283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:51260"
level=debug ts=2026-04-20T18:57:19.455324602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:57:29.596959403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:33964"
ts=2026-04-20T18:57:35.518716758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T18:57:42.817139473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:57:46.36517738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:60200"
ts=2026-04-20T18:57:49.163267668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58038"
level=debug ts=2026-04-20T18:58:03.172377216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:58:05.521692394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T18:58:25.581391221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T18:58:25.583197352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T18:58:29.602062899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:54958"
ts=2026-04-20T18:58:35.524655611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T18:58:46.370188422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:37612"
ts=2026-04-20T18:58:59.605622167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:51220"
ts=2026-04-20T18:59:05.527692835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T18:59:07.523519115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T18:59:16.373250492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:36634"
ts=2026-04-20T18:59:29.608331009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:36626"
ts=2026-04-20T18:59:35.530503408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T18:59:46.376894819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:57414"
ts=2026-04-20T18:59:59.611526489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:57508"
ts=2026-04-20T19:00:05.533443362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:00:11.951031563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:00:29.614511901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:56160"
ts=2026-04-20T19:00:35.53646931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:00:49.17679201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54530"
ts=2026-04-20T19:00:59.617422818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:42124"
ts=2026-04-20T19:01:05.539212205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:01:16.383006553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:58884"
level=debug ts=2026-04-20T19:01:17.472121217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:01:35.541912079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:02:05.545126934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:02:33.713057573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:02:35.547839071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:02:49.187303932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59820"
ts=2026-04-20T19:03:05.551168184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:03:19.189985747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33638"
ts=2026-04-20T19:03:29.627462648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:57596"
ts=2026-04-20T19:03:35.55435532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:03:46.392291282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:48722"
level=debug ts=2026-04-20T19:03:46.689647017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T19:03:46.691431607Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:03:59.630713266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:46432"
ts=2026-04-20T19:04:05.557537115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:04:22.46354222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:04:35.560461725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:04:46.39659886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:57034"
ts=2026-04-20T19:04:59.635876812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:46186"
ts=2026-04-20T19:05:05.563153526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:05:09.398082353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:05:35.565506792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:05:46.401800307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:40726"
level=debug ts=2026-04-20T19:05:46.8462261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:06:05.568381004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:06:16.405215355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:34542"
ts=2026-04-20T19:06:19.20399946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33968"
level=debug ts=2026-04-20T19:06:31.718154968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:06:35.571730461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:06:49.207058785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:42594"
level=info ts=2026-04-20T19:06:54.37583655Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T19:06:54.379306943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:06:54.381178654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:06:54.382841592Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:06:54.382871067Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T19:06:54.382883447Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T19:07:05.574399652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:07:16.410669059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:43744"
level=debug ts=2026-04-20T19:07:18.8472309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:07:18.849070154Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:07:35.576944986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:07:46.413623443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:41942"
ts=2026-04-20T19:08:05.57979007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:08:11.112367213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:08:35.582409552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:08:59.071623093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:08:59.652377754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:49532"
ts=2026-04-20T19:09:05.585528268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:09:35.588490375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:09:46.422964129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:52346"
level=debug ts=2026-04-20T19:09:55.262672366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:09:59.656583613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:44444"
ts=2026-04-20T19:10:05.591944119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:10:19.224376934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57424"
ts=2026-04-20T19:10:35.595186037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:10:46.428356996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:58394"
level=debug ts=2026-04-20T19:10:54.800749565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:11:05.598030205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:11:19.229261202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56658"
ts=2026-04-20T19:11:35.601543273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:11:42.101535792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:11:42.103771191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:11:59.664470549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:57196"
ts=2026-04-20T19:12:05.60523667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:12:19.234218272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35390"
ts=2026-04-20T19:12:35.608591863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:12:41.085849697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:12:49.237266859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38650"
ts=2026-04-20T19:13:05.612104971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:13:19.240343918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49646"
level=debug ts=2026-04-20T19:13:26.734050913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:13:35.615082028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:13:49.243385784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59214"
ts=2026-04-20T19:14:05.618169478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:14:19.246409191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:50914"
level=debug ts=2026-04-20T19:14:20.068674878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:14:29.674899015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:32808"
ts=2026-04-20T19:14:35.621648562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:14:59.677475238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:59168"
ts=2026-04-20T19:15:05.624433043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:15:17.544808617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:15:35.627453304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:15:59.682392107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:54440"
level=debug ts=2026-04-20T19:16:00.592537284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:16:00.594052063Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:16:05.630195937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:16:16.448925026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:45850"
ts=2026-04-20T19:16:35.632987475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:16:50.527355272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:16:54.387141555Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:16:54.388812506Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:16:54.390812625Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:16:54.390864163Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:17:05.63664016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:17:19.259536191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52766"
level=debug ts=2026-04-20T19:17:26.342393124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:17:35.639642372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:17:59.691628045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:58064"
ts=2026-04-20T19:18:05.643988203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:18:19.264681785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58852"
level=debug ts=2026-04-20T19:18:20.492937277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:18:35.647302665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:18:49.26740959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53504"
ts=2026-04-20T19:18:59.697014633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:35310"
ts=2026-04-20T19:19:05.649747524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:19:16.022668864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:19:35.652498314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:19:46.462426358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:39824"
ts=2026-04-20T19:19:49.271624369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41694"
ts=2026-04-20T19:19:59.702599413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:49484"
ts=2026-04-20T19:20:05.656111567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:20:17.337160542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:20:17.33949739Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:20:35.659414454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:20:49.276780015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38522"
ts=2026-04-20T19:20:59.706565842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:48612"
level=debug ts=2026-04-20T19:21:02.310940702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:21:05.662240124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:21:29.710205689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:48558"
ts=2026-04-20T19:21:35.664760306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:21:46.471330917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:35858"
ts=2026-04-20T19:21:49.282079437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33152"
level=debug ts=2026-04-20T19:22:00.292625543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:22:05.668358619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:22:19.285292647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57508"
ts=2026-04-20T19:22:35.671239511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:22:57.85566315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:23:05.674465702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:23:19.289850001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57958"
ts=2026-04-20T19:23:35.677679944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:23:36.421807393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:23:59.719218269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:41138"
ts=2026-04-20T19:24:05.681247975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:24:19.295268226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41674"
level=debug ts=2026-04-20T19:24:19.35550728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:24:19.357355315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:24:35.684151506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:24:46.481981633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:38768"
ts=2026-04-20T19:24:49.297932813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35034"
ts=2026-04-20T19:24:59.723630518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:48338"
ts=2026-04-20T19:25:05.687152374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:25:17.36959738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:25:35.690177844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:25:46.487177227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:42588"
level=debug ts=2026-04-20T19:26:02.772364018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:26:05.693379382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:26:35.696932493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:26:41.008127485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:26:54.38685908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:26:54.388654503Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:26:54.390298161Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:26:54.390324763Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:26:59.731074513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:48746"
ts=2026-04-20T19:27:05.700380776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:27:19.310678493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:36134"
ts=2026-04-20T19:27:35.703334785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:27:38.295350413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:27:59.736091541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:58542"
ts=2026-04-20T19:28:05.706381119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:28:26.547921261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:28:26.549956008Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:28:35.709181955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:28:49.318460014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41016"
level=debug ts=2026-04-20T19:29:02.219269257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:29:05.712970024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:29:29.74352231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:57970"
ts=2026-04-20T19:29:35.716459975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:29:41.285051797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:29:49.322962765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:50660"
ts=2026-04-20T19:29:59.747248823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:40790"
ts=2026-04-20T19:30:05.719472151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:30:18.668453506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:30:35.722771153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:30:59.752290004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:39626"
level=debug ts=2026-04-20T19:31:00.05862457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:31:05.725510556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:31:35.72806759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:31:49.332981329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:46582"
level=debug ts=2026-04-20T19:32:01.509380751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:32:01.511055348Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:32:05.731189189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:32:19.335873922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48848"
ts=2026-04-20T19:32:35.733751909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:32:46.519639138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:60400"
level=debug ts=2026-04-20T19:32:59.569948256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:33:05.736920534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:33:29.762028281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:59742"
ts=2026-04-20T19:33:35.739649992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:33:49.342980944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48342"
level=debug ts=2026-04-20T19:33:58.09330003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:33:59.764873805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:41926"
ts=2026-04-20T19:34:05.742443222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:34:19.346170351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:60502"
ts=2026-04-20T19:34:29.768675129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:52940"
ts=2026-04-20T19:34:35.745398187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:34:46.528377779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:46984"
level=debug ts=2026-04-20T19:34:49.672087889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:34:59.771370498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:57312"
ts=2026-04-20T19:35:05.748107341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:35:16.531097047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:38892"
level=debug ts=2026-04-20T19:35:26.950769894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:35:35.751402431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:35:46.534264118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:36222"
ts=2026-04-20T19:35:49.35355337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41428"
ts=2026-04-20T19:36:05.754783213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:36:16.536875407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:56962"
level=debug ts=2026-04-20T19:36:22.866609413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:36:22.868587246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:36:35.757794997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=error ts=2026-04-20T19:36:54.387203615Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:36:54.389218379Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:36:54.39088533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:36:54.390915451Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:37:05.760923797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:37:08.330796311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:37:29.783369574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:52754"
ts=2026-04-20T19:37:35.763669827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:37:51.020852539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:37:59.786068727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:45330"
ts=2026-04-20T19:38:05.76695694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:38:29.789049335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:51824"
ts=2026-04-20T19:38:35.770321851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:38:36.276025603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:38:49.365672698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:50222"
ts=2026-04-20T19:38:59.791636157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:56486"
ts=2026-04-20T19:39:05.773493005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:39:19.36857923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58444"
level=debug ts=2026-04-20T19:39:27.915232811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:39:29.794909453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:58106"
ts=2026-04-20T19:39:35.776684869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:39:49.371678306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49046"
ts=2026-04-20T19:40:05.779626346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:40:16.522906673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:40:16.524519801Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:40:16.550681295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:42464"
ts=2026-04-20T19:40:35.782471709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:40:49.376838862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41706"
ts=2026-04-20T19:40:59.802232383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:39978"
ts=2026-04-20T19:41:05.785998473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
level=debug ts=2026-04-20T19:41:16.382105316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:41:35.789483082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:42:05.793121433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
level=debug ts=2026-04-20T19:42:12.685510017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:42:29.809380845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:36496"
ts=2026-04-20T19:42:35.796245244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:42:49.388117412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:39668"
level=debug ts=2026-04-20T19:42:53.044338608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:43:05.799568323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:43:19.390477927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37248"
level=debug ts=2026-04-20T19:43:28.638075077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:43:35.802447225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
ts=2026-04-20T19:43:59.814923934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.101:52792"
ts=2026-04-20T19:44:05.805256298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:44:27.984103607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:44:27.985656364Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:44:35.808266359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-29c28a1e 10.129.0.101:7946"
ts=2026-04-20T19:44:46.568545715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:59324"
ts=2026-04-20T19:44:49.397218961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53712"
ts=2026-04-20T19:45:05.811024466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:45:16.572077859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.98:51604"
ts=2026-04-20T19:45:19.400567474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:39008"
level=debug ts=2026-04-20T19:45:19.951254304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:45:35.814394313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h9892-6b84359c 10.129.0.98:7946"
ts=2026-04-20T19:46:05.816941652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2jvrr-ab3f75e7 10.128.0.67:7946"
level=debug ts=2026-04-20T19:46:09.384694531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"