level=info ts=2026-05-09T14:45:46.972504Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-09T14:45:46.972537436Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-09T14:45:46.973828549Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-09T14:45:46.974257208Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T14:45:46.97434419Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=debug ts=2026-05-09T14:45:46.975839852Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T14:45:46.975922659Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=info ts=2026-05-09T14:45:46.975975047Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-23a0b45b
ts=2026-05-09T14:45:46.976079346Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-09T14:45:46.976096278Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=debug ts=2026-05-09T14:45:46.976111765Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=info ts=2026-05-09T14:45:46.976224048Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-09T14:45:46.982595434Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T14:45:46.982614193Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:45:46.982626785Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.404442ms
level=error ts=2026-05-09T14:45:46.982638949Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-09T14:45:46.982655368Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-09T14:45:46.986114575Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:46.986129113Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T14:45:46.992369013Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-09T14:45:46.992419496Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-09T14:45:46.992426576Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-09T14:45:46.99243384Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-09T14:45:46.992438477Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-09T14:45:46.992496654Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-09T14:45:46.992517903Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-09T14:45:46.992541572Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-09T14:45:46.992667254Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-09T14:45:46.992674983Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-09T14:45:46.992732616Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-09T14:45:46.992741943Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-09T14:45:46.992765933Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-09T14:45:46.993023487Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-09T14:45:46.993183365Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-09T14:45:46.993338589Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-09T14:45:47.994410674Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-09T14:45:48.110798747Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-09T14:45:48.110869334Z caller=loki.go:615 msg="Loki started" startup_time=1.204153894s
ts=2026-05-09T14:45:48.371521405Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:48.371541344Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:45:52.316520441Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:52.316542838Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T14:45:53.111491599Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-09T14:45:53.111540645Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-09T14:46:00.091484746Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:46:00.091509066Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:46:04.161972198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38546"
level=debug ts=2026-05-09T14:46:05.159067833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:46:08.674129156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42306"
ts=2026-05-09T14:46:08.687887876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.95:7946"
ts=2026-05-09T14:46:08.690433211Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.98:7946: dial tcp 10.129.0.98:7946: connect: connection refused"
ts=2026-05-09T14:46:08.691295414Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.65:7946: dial tcp 10.130.0.65:7946: connect: connection refused"
ts=2026-05-09T14:46:08.691388127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.66:7946"
ts=2026-05-09T14:46:08.691420673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41772"
ts=2026-05-09T14:46:08.692784087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-05-09T14:46:08.694262894Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
level=info ts=2026-05-09T14:46:08.694274152Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.711604163s
ts=2026-05-09T14:46:10.680101147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35070"
level=debug ts=2026-05-09T14:46:16.980227469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:46:23.11737956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60292"
level=debug ts=2026-05-09T14:46:39.902362782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:46:45.92202669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:47:01.964268785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:47:15.925678742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T14:47:23.120870208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49676"
level=debug ts=2026-05-09T14:47:27.780028035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T14:47:27.781668099Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T14:47:44.482093842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45886"
ts=2026-05-09T14:47:45.928297088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T14:48:09.958791841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:48:15.929796764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:48:44.487755192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42600"
ts=2026-05-09T14:48:45.931498987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:49:04.794501211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:49:15.933002291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:49:23.129211189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49728"
ts=2026-05-09T14:49:45.934919384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T14:50:15.937873131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T14:50:23.133229871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58140"
level=debug ts=2026-05-09T14:50:24.049163618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:50:45.941032007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T14:50:53.134206882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51922"
ts=2026-05-09T14:51:02.709444579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43542"
ts=2026-05-09T14:51:14.504886901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43996"
ts=2026-05-09T14:51:15.943040761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:51:32.71155942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42650"
level=debug ts=2026-05-09T14:51:39.938716504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:51:44.507632619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40406"
ts=2026-05-09T14:51:45.944342812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T14:52:15.945967934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:52:23.141551176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48764"
level=debug ts=2026-05-09T14:52:32.095222072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T14:52:32.096792208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T14:52:45.947690319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T14:53:02.720360832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33492"
ts=2026-05-09T14:53:14.517535863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42548"
ts=2026-05-09T14:53:15.949268862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T14:53:29.656785902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:53:32.722692225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56884"
ts=2026-05-09T14:53:44.521427263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:48634"
ts=2026-05-09T14:53:45.950907193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:53:53.148936211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58570"
level=debug ts=2026-05-09T14:54:11.604360732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:54:15.95174503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:54:45.953494897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T14:54:46.017997727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:55:15.95505209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:55:21.582485268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:55:23.155764166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54078"
ts=2026-05-09T14:55:32.733468751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60618"
ts=2026-05-09T14:55:45.958299267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T14:55:53.112494488Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-09T14:55:53.116184215Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T14:55:53.11822092Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T14:55:53.12001295Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T14:55:53.120036726Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T14:55:53.120048611Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-09T14:55:53.158259955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57844"
level=debug ts=2026-05-09T14:55:56.902208443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T14:55:56.904083462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T14:56:02.735907906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36570"
ts=2026-05-09T14:56:14.539601235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:52640"
ts=2026-05-09T14:56:15.961481086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T14:56:23.159920499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40188"
ts=2026-05-09T14:56:44.543298882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57526"
ts=2026-05-09T14:56:45.963216277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:56:48.17779132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:57:02.740322363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39224"
ts=2026-05-09T14:57:15.965148661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T14:57:23.165355502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47946"
ts=2026-05-09T14:57:32.742069363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33416"
level=debug ts=2026-05-09T14:57:42.394491408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:57:45.967318705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T14:58:02.745171961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37616"
ts=2026-05-09T14:58:15.969574377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T14:58:21.40842362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:58:32.747722218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41346"
ts=2026-05-09T14:58:45.972187396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T14:59:14.559842451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:56758"
ts=2026-05-09T14:59:15.975425286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:59:22.819209303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T14:59:32.752963333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42542"
ts=2026-05-09T14:59:45.978538166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T14:59:53.177887251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42808"
ts=2026-05-09T15:00:02.754891508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58616"
level=debug ts=2026-05-09T15:00:06.667980679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:00:06.669547055Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:00:15.980841665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:00:45.983266491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:00:46.372669609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:01:15.985603093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:01:23.185243803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40498"
ts=2026-05-09T15:01:32.761139503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43744"
level=debug ts=2026-05-09T15:01:40.001114165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:01:44.577354624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60422"
ts=2026-05-09T15:01:45.987823574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:01:53.186912918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44614"
ts=2026-05-09T15:02:15.98944064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:02:41.428439275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:02:45.991880562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:03:02.767404641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40198"
ts=2026-05-09T15:03:14.587777987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49734"
ts=2026-05-09T15:03:15.994959019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:03:16.982886239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:03:23.193076172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48038"
ts=2026-05-09T15:03:45.996924295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:03:58.235739918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:03:58.239184472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:04:02.772303119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40924"
ts=2026-05-09T15:04:15.998223412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:04:32.774223221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37720"
ts=2026-05-09T15:04:44.597518928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:48602"
ts=2026-05-09T15:04:46.000032945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:04:53.20046028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34616"
level=debug ts=2026-05-09T15:04:59.16989292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:05:14.601100424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42494"
ts=2026-05-09T15:05:16.002406941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:05:35.77653429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:05:46.004508451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=error ts=2026-05-09T15:05:53.129248182Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:05:53.131760901Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:05:53.133667018Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:05:53.133687031Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:05:53.205126244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60630"
level=debug ts=2026-05-09T15:06:15.23288132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:06:16.005888945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:06:46.007955016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:06:57.545699572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:07:02.785539404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45716"
ts=2026-05-09T15:07:16.010451409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:07:32.787932037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56830"
level=debug ts=2026-05-09T15:07:36.370404656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:07:36.372065847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:07:46.01355661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:08:02.790478341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40954"
level=debug ts=2026-05-09T15:08:11.309908617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:08:16.017034235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:08:32.800481041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34918"
ts=2026-05-09T15:08:44.624764816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37870"
ts=2026-05-09T15:08:46.020071673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:08:53.893980986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:09:14.627290279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:39850"
ts=2026-05-09T15:09:16.022771944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:09:23.220226203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37620"
ts=2026-05-09T15:09:32.806714189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36180"
level=debug ts=2026-05-09T15:09:33.791674099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:09:46.025356789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:10:16.026670674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:10:19.555264148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:10:46.031098094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:10:53.22617987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32884"
level=debug ts=2026-05-09T15:10:59.744236107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:10:59.74589517Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:11:16.033323292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:11:23.228516491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51694"
ts=2026-05-09T15:11:32.826159547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36234"
ts=2026-05-09T15:11:46.03610223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:11:48.088626636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:11:53.230518021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41368"
ts=2026-05-09T15:12:16.039824896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:12:25.450620915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:12:44.648767862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43742"
ts=2026-05-09T15:12:46.041887158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:13:06.299802316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:13:16.043193117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:13:41.949108462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:13:44.655215687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:52206"
ts=2026-05-09T15:13:46.045509084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:13:53.239695966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41472"
ts=2026-05-09T15:14:02.837869751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44532"
ts=2026-05-09T15:14:14.65831266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40008"
ts=2026-05-09T15:14:16.048670956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:14:23.241243469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58964"
level=debug ts=2026-05-09T15:14:30.542120735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:14:30.543698605Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:14:46.050756484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:15:08.370690099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:15:16.052662001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:15:32.84505469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40780"
ts=2026-05-09T15:15:46.054001996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:15:47.681146202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:15:53.127999068Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:15:53.135027867Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:15:53.13776705Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:15:53.137832976Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:16:16.056122265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:16:23.251589987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53004"
level=debug ts=2026-05-09T15:16:34.322500977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:16:46.058121362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:17:14.677606305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45026"
ts=2026-05-09T15:17:16.059456021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:17:19.857229677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:17:23.255106026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37774"
ts=2026-05-09T15:17:44.680893489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:50086"
ts=2026-05-09T15:17:46.06193573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:17:53.257888595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52414"
level=debug ts=2026-05-09T15:18:12.753750591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:18:12.755644676Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:18:16.064665914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:18:23.259082627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43940"
ts=2026-05-09T15:18:46.067636578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:18:53.260598849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51294"
level=debug ts=2026-05-09T15:19:10.295400092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:19:14.689177767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53606"
ts=2026-05-09T15:19:16.071325679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:19:46.074319433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:20:09.886218132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:20:16.077283832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:20:23.265490503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34038"
ts=2026-05-09T15:20:32.86911875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54880"
ts=2026-05-09T15:20:44.70257865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43670"
ts=2026-05-09T15:20:46.080804509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:20:46.579764464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:21:16.083387727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:21:28.061430465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:21:46.085146626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:21:53.270973178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57712"
ts=2026-05-09T15:22:14.712401869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35422"
ts=2026-05-09T15:22:16.086464902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:22:16.864085669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:22:16.865574142Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:22:32.877913697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39618"
ts=2026-05-09T15:22:46.088028429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:23:14.719270553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59144"
ts=2026-05-09T15:23:16.0908439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:23:17.474413129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:23:23.276948455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48228"
ts=2026-05-09T15:23:46.093199838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:23:54.902341257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:24:14.726950517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:48862"
ts=2026-05-09T15:24:16.094852415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:24:23.280360983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60432"
level=debug ts=2026-05-09T15:24:41.555268773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:24:46.097400374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:25:02.890206484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44842"
ts=2026-05-09T15:25:16.099598957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:25:23.283350166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55090"
level=debug ts=2026-05-09T15:25:27.064914724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:25:44.739970901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:54650"
ts=2026-05-09T15:25:46.101826466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=error ts=2026-05-09T15:25:53.142488736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:25:53.149179183Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:25:53.151895524Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:25:53.151928044Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:25:53.284857011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57150"
ts=2026-05-09T15:26:14.74395835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49038"
ts=2026-05-09T15:26:16.105446155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:26:26.166676964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:26:26.168357199Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:26:32.897047277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58768"
ts=2026-05-09T15:26:46.107363944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:27:03.448750966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:27:16.108804753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:27:23.293954045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39362"
ts=2026-05-09T15:27:44.753390487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34208"
ts=2026-05-09T15:27:46.110642111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:28:02.911727052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:28:16.112009652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:28:46.114111807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:28:59.029379556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:29:16.11616936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:29:39.270385395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:29:46.117654585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:29:53.304562035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44236"
ts=2026-05-09T15:30:16.119797505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:30:28.983959137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:30:28.98540064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:30:44.773223079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:39422"
ts=2026-05-09T15:30:46.122684485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:31:02.916650101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39166"
ts=2026-05-09T15:31:14.775874311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35974"
ts=2026-05-09T15:31:16.12506557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:31:19.786878595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:31:44.779295964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42120"
ts=2026-05-09T15:31:46.127390318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:32:02.920453466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:32:16.128725213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:32:23.314591594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58134"
ts=2026-05-09T15:32:32.924499333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44810"
ts=2026-05-09T15:32:46.130158612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:32:53.315664348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55696"
level=debug ts=2026-05-09T15:33:01.441949918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:33:02.926495396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48026"
ts=2026-05-09T15:33:16.131885183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:33:32.928932906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60852"
ts=2026-05-09T15:33:46.133966049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:33:58.380305191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:34:02.930777533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42358"
ts=2026-05-09T15:34:16.135541181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:34:32.932964407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40340"
level=debug ts=2026-05-09T15:34:45.41467609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:34:45.416094927Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:34:46.138148142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:34:53.32239217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36012"
ts=2026-05-09T15:35:14.80113883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33020"
ts=2026-05-09T15:35:16.140414668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:35:46.142824531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:35:46.364721536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:35:53.123012579Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:35:53.124733205Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:35:53.126383488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:35:53.126410675Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:35:53.326771389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48736"
ts=2026-05-09T15:36:16.144061971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:36:28.01968543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:36:46.145666407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:37:02.943142701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49464"
level=debug ts=2026-05-09T15:37:07.758234729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:37:14.813232883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:58524"
ts=2026-05-09T15:37:16.148078357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:37:23.333957713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59406"
ts=2026-05-09T15:37:32.945460363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48482"
ts=2026-05-09T15:37:44.816597943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:55522"
ts=2026-05-09T15:37:46.150798696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:38:02.947478039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45626"
level=debug ts=2026-05-09T15:38:06.558030453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:38:16.153174089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:38:32.949091762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49090"
ts=2026-05-09T15:38:44.823197507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:56064"
ts=2026-05-09T15:38:46.156190827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:38:46.784652644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:38:46.78628546Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:39:02.951423234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49338"
ts=2026-05-09T15:39:16.159005595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:39:23.341150213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52500"
level=debug ts=2026-05-09T15:39:24.175164618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:39:46.162212494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:40:02.213505175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:40:16.165285714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:40:32.959073809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51748"
ts=2026-05-09T15:40:44.837022414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42936"
ts=2026-05-09T15:40:46.168468209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:40:53.348149585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36954"
level=debug ts=2026-05-09T15:40:57.365558165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:41:02.961429308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56712"
ts=2026-05-09T15:41:16.170793801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:41:23.349259314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45728"
ts=2026-05-09T15:41:32.963647599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48462"
ts=2026-05-09T15:41:46.1724748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:41:57.611371825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:42:02.966048343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47286"
ts=2026-05-09T15:42:16.17453059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:42:36.564788457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:42:36.566524668Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:42:46.177111222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:42:53.35479347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45544"
ts=2026-05-09T15:43:14.852146984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36866"
ts=2026-05-09T15:43:16.179982428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:43:29.870967739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:43:44.855778379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:51346"
ts=2026-05-09T15:43:46.181891192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:44:16.184348187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:44:18.941496933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:44:46.187054516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:44:53.36201189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49630"
level=debug ts=2026-05-09T15:45:09.313889984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:45:16.190526251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:45:23.363295578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40526"
ts=2026-05-09T15:45:32.982902883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53056"
ts=2026-05-09T15:45:46.193504207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=error ts=2026-05-09T15:45:53.124599012Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:45:53.126668908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:45:53.128494043Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:45:53.128513769Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T15:45:59.818547307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:46:14.873145869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59838"
ts=2026-05-09T15:46:16.195305391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:46:23.366439258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48746"
ts=2026-05-09T15:46:44.876670873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35352"
ts=2026-05-09T15:46:46.196853928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:46:49.675239668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:46:49.676645954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:46:53.368087425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58812"
ts=2026-05-09T15:47:14.879932758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43834"
ts=2026-05-09T15:47:16.198413576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:47:23.369051471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51080"
level=debug ts=2026-05-09T15:47:30.629350986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:47:32.990198897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34140"
ts=2026-05-09T15:47:46.20098565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:48:14.258972904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:48:14.887269423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35868"
ts=2026-05-09T15:48:16.20324381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:48:46.205531481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:48:53.376032935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52226"
ts=2026-05-09T15:49:02.99877276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37960"
level=debug ts=2026-05-09T15:49:15.154794177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:49:16.208471196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:49:46.209526897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:49:51.854353791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:50:16.211665974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:50:23.381173425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46320"
ts=2026-05-09T15:50:33.011969968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47424"
ts=2026-05-09T15:50:46.214031525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:50:46.506091952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:50:46.50764496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:51:03.01336653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38172"
ts=2026-05-09T15:51:16.216343157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:51:31.821331228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:51:46.218619022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:51:53.385788417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47822"
level=debug ts=2026-05-09T15:52:06.96340051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:52:16.221632842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:52:33.02106831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42532"
ts=2026-05-09T15:52:46.224602445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:52:53.389261249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52832"
level=debug ts=2026-05-09T15:53:01.500628582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:53:16.227883995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T15:53:23.390607259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41298"
ts=2026-05-09T15:53:33.026445219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42732"
level=debug ts=2026-05-09T15:53:42.622397822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:53:44.92270982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:56012"
ts=2026-05-09T15:53:46.230921795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:54:16.23626298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:54:24.845183202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:54:24.846977629Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:54:46.23794416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:54:53.395988103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44950"
ts=2026-05-09T15:55:03.033302026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50548"
ts=2026-05-09T15:55:14.933236579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43250"
ts=2026-05-09T15:55:16.239379968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T15:55:21.052324501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:55:46.240931227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=error ts=2026-05-09T15:55:53.123641021Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:55:53.125395201Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:55:53.126931332Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T15:55:53.12695046Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:56:03.036838127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46728"
level=debug ts=2026-05-09T15:56:05.862175103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:56:16.242299679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:56:23.401108873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59244"
ts=2026-05-09T15:56:46.244018161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:56:53.401925905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56164"
level=debug ts=2026-05-09T15:56:56.566843679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:57:14.946641369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37498"
ts=2026-05-09T15:57:16.246222337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:57:35.659978316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:57:46.247955553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:57:53.405093296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47720"
ts=2026-05-09T15:58:16.250370757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:58:27.915539315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:58:27.917196729Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T15:58:46.254015707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T15:58:53.409790672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33808"
ts=2026-05-09T15:59:03.051730628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52764"
ts=2026-05-09T15:59:16.25705664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:59:18.57566639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:59:46.259431545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T15:59:53.412984739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55570"
level=debug ts=2026-05-09T16:00:12.116830597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:00:14.965575064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49928"
ts=2026-05-09T16:00:16.261649708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:00:23.413965244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43924"
ts=2026-05-09T16:00:46.263729647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:00:53.415009405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33110"
level=debug ts=2026-05-09T16:01:11.794875347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:01:16.266189371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:01:46.268773603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:01:53.419123264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47328"
level=debug ts=2026-05-09T16:02:00.759029967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:02:03.067368399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35398"
ts=2026-05-09T16:02:16.271218296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:02:33.069452095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43534"
level=debug ts=2026-05-09T16:02:45.808269284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:02:45.809822508Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:02:46.273346693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:02:53.422077049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56298"
ts=2026-05-09T16:03:16.275069469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:03:23.423061255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59740"
level=debug ts=2026-05-09T16:03:37.944766145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:03:46.2770603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:03:53.424777862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34840"
ts=2026-05-09T16:04:16.278537854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:04:20.445532863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:04:23.425984625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60906"
ts=2026-05-09T16:04:44.992915612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49034"
ts=2026-05-09T16:04:46.280205722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:05:03.082012852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40852"
level=debug ts=2026-05-09T16:05:08.463948042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:05:16.283021933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:05:23.429958743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37620"
ts=2026-05-09T16:05:33.084339728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55552"
ts=2026-05-09T16:05:46.286710684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:05:48.716766804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:05:53.122800162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:05:53.124380537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:05:53.125866952Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:05:53.125889049Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:06:16.288852396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:06:23.433752489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42072"
level=debug ts=2026-05-09T16:06:36.359645784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:06:36.361097212Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:06:46.29067663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:06:53.435641538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44588"
ts=2026-05-09T16:07:16.293290482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:07:28.519647034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:07:45.012545011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:46712"
ts=2026-05-09T16:07:46.296764201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:07:53.4389348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53366"
level=debug ts=2026-05-09T16:08:05.06176268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:08:15.017957811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35148"
ts=2026-05-09T16:08:16.299282559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:08:33.09996429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46814"
ts=2026-05-09T16:08:45.022010021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36080"
ts=2026-05-09T16:08:46.302179231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:08:51.895557157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:09:16.304339261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:09:38.425459429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:09:46.30722645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:10:15.037680971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41128"
ts=2026-05-09T16:10:16.309533422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:10:23.456346652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52530"
ts=2026-05-09T16:10:33.109677255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36952"
level=debug ts=2026-05-09T16:10:39.427419355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:10:39.429028096Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:10:46.311396126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:10:53.457993165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52900"
ts=2026-05-09T16:11:15.044450245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34654"
ts=2026-05-09T16:11:16.313272798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:11:21.810737134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:11:23.459851283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43146"
ts=2026-05-09T16:11:33.113890942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47244"
ts=2026-05-09T16:11:46.315139299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:11:53.460923467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48766"
level=debug ts=2026-05-09T16:12:05.130661515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:12:15.050453303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:50810"
ts=2026-05-09T16:12:16.317756908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:12:33.117613301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54938"
ts=2026-05-09T16:12:45.053155735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40042"
ts=2026-05-09T16:12:46.320071788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:12:50.773513305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:13:15.056281136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57250"
ts=2026-05-09T16:13:16.32226547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:13:23.468430798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43642"
ts=2026-05-09T16:13:46.32573978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:13:48.837527806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:14:15.062957538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36084"
ts=2026-05-09T16:14:16.328091112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:14:28.922347703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:14:28.923917124Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:14:46.330478252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:15:03.128088319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53772"
level=debug ts=2026-05-09T16:15:03.569478728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:15:16.339948199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:15:46.34586766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:15:52.498214235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:15:53.1226706Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:15:53.124298613Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:15:53.125894485Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T16:15:53.125914525Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:16:16.348958584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:16:44.281634432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:16:46.351387724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:16:53.489978564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60852"
ts=2026-05-09T16:17:15.083568306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37608"
ts=2026-05-09T16:17:16.354377307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
ts=2026-05-09T16:17:23.491139548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52986"
level=debug ts=2026-05-09T16:17:33.542768485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:17:46.356859916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:18:16.358579698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:18:23.494682754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41542"
level=debug ts=2026-05-09T16:18:25.831666697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:18:25.834492932Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T16:18:46.360688738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
ts=2026-05-09T16:19:16.362519547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f2a52d4 10.130.0.64:7946"
level=debug ts=2026-05-09T16:19:20.586462257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T16:19:33.150549761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51288"
ts=2026-05-09T16:19:45.107209586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:55828"
ts=2026-05-09T16:19:46.364440858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
ts=2026-05-09T16:19:53.50138512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41946"
ts=2026-05-09T16:20:16.366885119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:20:20.256088165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"