level=info ts=2026-05-07T11:02:17.208221518Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T11:02:17.208248929Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T11:02:17.209940805Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T11:02:17.210322704Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T11:02:17.210428838Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.53 inf=eth0
level=debug ts=2026-05-07T11:02:17.211073709Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T11:02:17.211148589Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.53 inf=eth0
level=info ts=2026-05-07T11:02:17.211198634Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-7166c551
ts=2026-05-07T11:02:17.211290889Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T11:02:17.211304062Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.53 advertisePort=7946
level=debug ts=2026-05-07T11:02:17.211325632Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.53 advertisePort=7946
level=info ts=2026-05-07T11:02:17.21145388Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T11:02:17.219185611Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T11:02:17.219205474Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T11:02:17.219219681Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.7683ms
level=error ts=2026-05-07T11:02:17.219231542Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T11:02:17.219246061Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T11:02:17.222445301Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T11:02:17.222461048Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T11:02:17.225905407Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-07T11:02:17.225919892Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-07T11:02:17.225918474Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T11:02:17.225936583Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T11:02:17.225948047Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-07T11:02:17.22593791Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T11:02:17.226026587Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-07T11:02:17.226032417Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-07T11:02:17.22627363Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T11:02:17.226296195Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T11:02:17.226382836Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T11:02:17.22639702Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T11:02:17.226413838Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T11:02:17.226529735Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T11:02:17.365208014Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T11:02:17.365278179Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-07T11:02:18.366060477Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-07T11:02:18.41680167Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T11:02:18.41682355Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T11:02:18.565126492Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T11:02:18.565178816Z caller=loki.go:615 msg="Loki started" startup_time=1.417603066s
ts=2026-05-07T11:02:22.250003554Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T11:02:22.250036883Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T11:02:23.565386875Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T11:02:23.565438598Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T11:02:29.587057409Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T11:02:29.587094473Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T11:02:33.995822834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:02:40.456424964Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-05-07T11:02:40.457792698Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
level=warn ts=2026-05-07T11:02:40.457809645Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="2 errors occurred:\n\t* failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused\n\t* failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused\n\n"
ts=2026-05-07T11:02:47.619075039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51616"
level=debug ts=2026-05-07T11:02:57.373461159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:02:57.805276231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55206"
ts=2026-05-07T11:02:57.899014135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58496"
ts=2026-05-07T11:03:03.726180857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.71:7946"
ts=2026-05-07T11:03:03.729178974Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-05-07T11:03:03.730558387Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
ts=2026-05-07T11:03:03.730688878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.50:7946"
ts=2026-05-07T11:03:03.732280942Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.52:7946: dial tcp 10.130.0.52:7946: connect: connection refused"
ts=2026-05-07T11:03:03.732405921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.53:7946"
ts=2026-05-07T11:03:03.732474088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:36194"
ts=2026-05-07T11:03:03.733379171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
level=info ts=2026-05-07T11:03:03.734236819Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=46.514984142s
ts=2026-05-07T11:03:16.925083365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:03:23.878449294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:03:46.926826778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:03:49.997383614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:04:14.053670526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36402"
level=debug ts=2026-05-07T11:04:14.706936023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T11:04:14.708678949Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:04:16.928984455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:04:46.931622429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:04:48.650229682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46176"
level=debug ts=2026-05-07T11:05:01.220686747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:05:16.93335682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:05:44.059823398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42616"
ts=2026-05-07T11:05:46.934622019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:05:55.689188024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:06:14.060986763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:47050"
ts=2026-05-07T11:06:16.936691438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:06:18.658551544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55390"
ts=2026-05-07T11:06:31.861560777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45410"
ts=2026-05-07T11:06:44.061983921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42472"
ts=2026-05-07T11:06:46.939223155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:07:01.864871444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47800"
level=debug ts=2026-05-07T11:07:05.53082384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:07:16.941518571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:07:31.867893904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49576"
ts=2026-05-07T11:07:46.944080376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:07:48.665915432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54424"
level=debug ts=2026-05-07T11:08:12.25350331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T11:08:16.946423048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:08:18.668094259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49642"
ts=2026-05-07T11:08:46.948033356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:09:14.074280199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34310"
ts=2026-05-07T11:09:16.949746927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:09:18.674101371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40710"
level=debug ts=2026-05-07T11:09:30.938557683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T11:09:30.940064616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:09:46.95053379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:09:48.676117099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58028"
ts=2026-05-07T11:10:14.078404926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:40464"
ts=2026-05-07T11:10:16.953097206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:10:33.535024161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:10:46.955505467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:11:01.889237005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:44980"
ts=2026-05-07T11:11:14.082094971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50556"
ts=2026-05-07T11:11:16.958016949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:11:18.683354001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57058"
level=debug ts=2026-05-07T11:11:30.104038781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:11:44.083520018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34354"
ts=2026-05-07T11:11:46.960731656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:12:16.96363971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:12:18.686716092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52924"
level=debug ts=2026-05-07T11:12:21.754119616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T11:12:23.565889624Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T11:12:23.567949527Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:12:23.569406568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:12:23.570795896Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:12:23.570818602Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T11:12:23.570830628Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T11:12:46.965879577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:12:48.68898584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46406"
level=debug ts=2026-05-07T11:13:13.365897372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:13:16.968098977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:13:18.694744518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51220"
ts=2026-05-07T11:13:31.903558087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38484"
ts=2026-05-07T11:13:46.971672341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:13:48.696488836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60810"
level=debug ts=2026-05-07T11:14:05.604468804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:14:05.60612202Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:14:16.974035004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:14:18.698932505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48150"
ts=2026-05-07T11:14:31.909856412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34600"
ts=2026-05-07T11:14:46.975573848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:14:48.701048597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54700"
ts=2026-05-07T11:15:01.912452154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:35274"
level=debug ts=2026-05-07T11:15:03.541835841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:15:16.977493478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:15:44.102522031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53414"
ts=2026-05-07T11:15:46.97897016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:15:54.672440792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:16:01.918273432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41846"
ts=2026-05-07T11:16:16.980518158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:16:44.107035154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55964"
ts=2026-05-07T11:16:46.983119169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:16:48.711349179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33366"
level=debug ts=2026-05-07T11:16:52.083936216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:17:14.108052358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52082"
ts=2026-05-07T11:17:16.985244682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:17:18.713642898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48658"
ts=2026-05-07T11:17:31.927485486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54904"
level=debug ts=2026-05-07T11:17:36.12611918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:17:44.109019516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46600"
ts=2026-05-07T11:17:46.986017956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:18:16.987775924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:18:18.71909052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47080"
ts=2026-05-07T11:18:31.934241526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:43624"
level=debug ts=2026-05-07T11:18:33.549645133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:18:33.551163051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:18:46.990766293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:19:01.936929542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41136"
ts=2026-05-07T11:19:16.993680394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:19:26.725071274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:19:31.940088932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38958"
ts=2026-05-07T11:19:44.118147871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43568"
ts=2026-05-07T11:19:46.995696516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:19:48.728799517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53228"
ts=2026-05-07T11:20:01.943128777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:44002"
level=debug ts=2026-05-07T11:20:11.429672323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:20:16.997153116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:20:44.122280963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:40224"
ts=2026-05-07T11:20:46.999912079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:21:01.399603611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:21:14.123273169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52278"
ts=2026-05-07T11:21:17.002545767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:21:31.953664471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52440"
ts=2026-05-07T11:21:47.005059565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:21:47.370235969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:21:48.737610658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53866"
ts=2026-05-07T11:22:14.126840593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:45558"
ts=2026-05-07T11:22:17.007685534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=error ts=2026-05-07T11:22:23.575224187Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:22:23.576871153Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:22:23.578516034Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:22:23.57853869Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:22:31.959932557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:36132"
level=debug ts=2026-05-07T11:22:39.581859979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:22:39.583498128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:22:47.00920376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:23:14.132175832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53526"
ts=2026-05-07T11:23:17.010511293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:23:31.966056673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:42918"
level=debug ts=2026-05-07T11:23:32.682898822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:23:47.012707639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:24:01.970897813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45058"
level=debug ts=2026-05-07T11:24:11.96393575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:24:14.136217466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39982"
ts=2026-05-07T11:24:17.015703635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:24:18.750080096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56902"
ts=2026-05-07T11:24:47.018410102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:24:56.316518473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:25:17.020707829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:25:18.754598205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37132"
ts=2026-05-07T11:25:47.023227512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:25:47.588428751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:25:48.756475161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43888"
ts=2026-05-07T11:26:17.025249254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:26:28.4247047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:26:28.426433071Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:26:44.152446054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:41566"
ts=2026-05-07T11:26:47.028523606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:26:48.7622845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47034"
ts=2026-05-07T11:27:01.992025977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48966"
ts=2026-05-07T11:27:17.031074994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:27:18.763946457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50626"
level=debug ts=2026-05-07T11:27:24.451333889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:27:44.159929521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52934"
ts=2026-05-07T11:27:47.032372352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:27:48.765584632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49158"
ts=2026-05-07T11:28:01.999126484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55052"
level=debug ts=2026-05-07T11:28:16.164339317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:28:17.033923551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:28:44.163298381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43360"
ts=2026-05-07T11:28:47.036203656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:29:10.016822472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:29:17.03828172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:29:47.042623323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:30:09.670133277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:30:14.174149403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33646"
ts=2026-05-07T11:30:17.049251544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:30:47.055392994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:30:47.225687424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:30:47.22721573Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:31:17.059256693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:31:32.022816292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:37454"
level=debug ts=2026-05-07T11:31:41.308518509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:31:47.062624376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:32:14.184166279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:41568"
ts=2026-05-07T11:32:17.064881026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:32:18.799466909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55276"
level=error ts=2026-05-07T11:32:23.575823838Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:32:23.577577868Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:32:23.579215113Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:32:23.579235888Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T11:32:25.905247304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:32:47.067912974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:33:17.07009768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:33:17.17343108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:33:47.073556755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:34:01.381344716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:34:14.194172709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42686"
ts=2026-05-07T11:34:17.075907868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:34:32.042914195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46706"
level=debug ts=2026-05-07T11:34:38.559623198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:34:38.562126765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:34:44.195837644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:54224"
ts=2026-05-07T11:34:47.078893849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:35:02.046556338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:40302"
ts=2026-05-07T11:35:17.081687919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:35:18.824000447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39762"
level=debug ts=2026-05-07T11:35:26.677089752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:35:47.085303852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:35:48.826165155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47942"
ts=2026-05-07T11:36:14.203037564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:54926"
ts=2026-05-07T11:36:17.087474482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:36:23.247559776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:36:47.09000188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:37:01.267093423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:37:17.092896209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:37:32.064864622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:36766"
ts=2026-05-07T11:37:44.210409759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:41294"
ts=2026-05-07T11:37:47.096533173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:37:48.835549479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53764"
level=debug ts=2026-05-07T11:38:01.999709385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:38:14.211246962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37830"
ts=2026-05-07T11:38:17.099805056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:38:47.102688896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:38:48.840832707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42806"
level=debug ts=2026-05-07T11:38:58.779856634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:38:58.781409329Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:39:17.105243703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:39:44.219467334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43078"
level=debug ts=2026-05-07T11:39:44.274582685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:39:47.107357606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:40:02.082484358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:51112"
ts=2026-05-07T11:40:17.109603092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:40:43.489406365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:40:47.112919597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:41:17.115803093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:41:22.850714291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:41:32.092088042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38406"
ts=2026-05-07T11:41:47.157347053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:41:58.153578337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:42:02.095434072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56582"
ts=2026-05-07T11:42:14.229488628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50892"
ts=2026-05-07T11:42:17.160199469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=error ts=2026-05-07T11:42:23.575845323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:42:23.577692383Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:42:23.579500624Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:42:23.579522784Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:42:47.162201883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:42:59.4425369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:42:59.445006596Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:43:14.232715167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53956"
ts=2026-05-07T11:43:17.164824149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:43:18.86573009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49268"
ts=2026-05-07T11:43:44.234428747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53158"
level=debug ts=2026-05-07T11:43:46.516602635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:43:47.168733416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:43:48.867476757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48022"
ts=2026-05-07T11:44:17.170710861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:44:18.869097944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39998"
level=debug ts=2026-05-07T11:44:34.052666045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:44:47.172156461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:45:02.114844754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56934"
ts=2026-05-07T11:45:17.17517673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:45:18.874970556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50830"
level=debug ts=2026-05-07T11:45:19.806214217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:45:32.117979442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60218"
ts=2026-05-07T11:45:44.242514738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:54692"
ts=2026-05-07T11:45:47.177342035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:45:48.877289046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56968"
ts=2026-05-07T11:46:02.121217862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:33444"
level=debug ts=2026-05-07T11:46:10.925450225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:46:14.243358824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:38512"
ts=2026-05-07T11:46:17.17925592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:46:47.182101888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:46:58.259274381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:46:58.261565171Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:47:17.184357267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:47:36.371821855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:47:47.186155378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:47:48.886877398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43890"
ts=2026-05-07T11:48:02.133962883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48822"
ts=2026-05-07T11:48:14.261285526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39598"
ts=2026-05-07T11:48:17.188194481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:48:26.725438248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:48:44.26313776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39664"
ts=2026-05-07T11:48:47.190688129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:49:14.265125294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42204"
ts=2026-05-07T11:49:17.193236549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:49:26.298060265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:49:32.143896726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45260"
ts=2026-05-07T11:49:44.267422231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:49100"
ts=2026-05-07T11:49:47.195292681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:50:17.196977624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:50:18.393230518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:50:18.897869011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58940"
ts=2026-05-07T11:50:44.270810229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36918"
ts=2026-05-07T11:50:47.198667106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:51:17.201105441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T11:51:17.954221499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:51:17.956127597Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:51:18.901178749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54924"
ts=2026-05-07T11:51:47.203366462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:52:02.160316379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56262"
level=debug ts=2026-05-07T11:52:09.783459266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:52:17.205889551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=error ts=2026-05-07T11:52:23.575209142Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:52:23.577014283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:52:23.578728888Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T11:52:23.578768422Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:52:32.163621469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49324"
ts=2026-05-07T11:52:47.208405515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T11:53:02.167030021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:43206"
level=debug ts=2026-05-07T11:53:10.378515531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:53:17.210324141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:53:47.212130693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:53:47.283576179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:53:48.91218899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36150"
ts=2026-05-07T11:54:17.214312938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:54:18.914676208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35306"
level=debug ts=2026-05-07T11:54:21.492705115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:54:47.217102646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:54:48.917129172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45846"
level=debug ts=2026-05-07T11:55:12.136338789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:55:12.138055729Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:55:17.219702581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:55:47.22097952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:55:49.020403794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:56:17.223340921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:56:37.097879005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:56:47.225563957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:56:48.926143822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57630"
ts=2026-05-07T11:57:14.304698759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55204"
ts=2026-05-07T11:57:17.227955015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T11:57:25.790002867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:57:44.306008678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:48188"
ts=2026-05-07T11:57:47.229881926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T11:57:48.931108123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51082"
ts=2026-05-07T11:58:02.200691917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38432"
ts=2026-05-07T11:58:14.306998884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46482"
ts=2026-05-07T11:58:17.232346006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:58:17.540241514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T11:58:18.933104424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55786"
ts=2026-05-07T11:58:47.233782038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T11:59:11.292658835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T11:59:11.294329022Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T11:59:17.236071723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T11:59:18.938037404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38008"
ts=2026-05-07T11:59:47.238248953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:00:03.840470637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:00:14.317181431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:56524"
ts=2026-05-07T12:00:17.244898434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:00:45.197354447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:00:47.246950013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:01:17.248586965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:01:30.187618874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:01:32.224009258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47524"
ts=2026-05-07T12:01:47.250037579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:02:06.874527474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:02:17.251548941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=error ts=2026-05-07T12:02:23.577350392Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:02:23.57930718Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:02:23.581186791Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:02:23.581210823Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:02:32.230089854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:59678"
ts=2026-05-07T12:02:44.327860151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:45846"
ts=2026-05-07T12:02:47.254038163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:02:48.678517341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:02:48.68010244Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:02:48.951914324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44062"
ts=2026-05-07T12:03:17.256089063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:03:32.236705574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:42388"
ts=2026-05-07T12:03:47.257581853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:03:50.121418165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:04:14.333888657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46558"
ts=2026-05-07T12:04:17.259872444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:04:18.959168736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59700"
ts=2026-05-07T12:04:32.24285342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54466"
ts=2026-05-07T12:04:47.264978465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:04:51.35946369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:05:17.267027863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:05:18.965585636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55286"
level=debug ts=2026-05-07T12:05:39.939979682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:05:47.268246383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:05:48.967907155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56006"
ts=2026-05-07T12:06:14.344171315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42754"
ts=2026-05-07T12:06:17.270724037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:06:31.235634072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:06:32.25523829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:51454"
ts=2026-05-07T12:06:44.345720232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36446"
ts=2026-05-07T12:06:47.27337281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:06:48.972773295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33058"
ts=2026-05-07T12:07:02.258572654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46368"
ts=2026-05-07T12:07:17.275920405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:07:29.929653732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:07:29.931109203Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:07:32.261317589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57682"
ts=2026-05-07T12:07:47.27749648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:07:48.975970175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58834"
ts=2026-05-07T12:08:17.279343448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:08:23.583927964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:08:47.280618159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:09:12.339093095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:09:14.358793332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:45806"
ts=2026-05-07T12:09:17.282911765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:09:44.360129539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:54778"
ts=2026-05-07T12:09:47.285914688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:09:56.751294454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:10:17.287446504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:10:32.278378143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34020"
level=debug ts=2026-05-07T12:10:44.684910014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:10:47.289495631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:11:17.29144922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:11:36.544371084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:11:36.545831618Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:11:47.292811373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:12:17.294894025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:12:18.998653713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35836"
level=error ts=2026-05-07T12:12:23.577283654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:12:23.579124492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:12:23.5807211Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:12:23.580745855Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T12:12:28.629262249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:12:32.291720228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58718"
ts=2026-05-07T12:12:47.296921175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:13:02.294824581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:44432"
level=debug ts=2026-05-07T12:13:07.63213835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:13:17.298945608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:13:42.032215562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:13:47.300962412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:14:17.30434832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:14:39.819619889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:14:44.384686853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46484"
ts=2026-05-07T12:14:47.307842032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:15:14.386171372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:56276"
ts=2026-05-07T12:15:17.308625613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:15:39.784427933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:15:39.78598715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:15:47.310801703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:15:49.014904149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33188"
ts=2026-05-07T12:16:17.312962073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:16:30.15573629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:16:44.392298174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44724"
ts=2026-05-07T12:16:47.314845304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:16:49.020738366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58314"
ts=2026-05-07T12:17:17.315633908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:17:31.714317215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:17:47.317599549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:17:49.024595209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51910"
ts=2026-05-07T12:18:14.399802973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:54822"
ts=2026-05-07T12:18:17.319735144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:18:18.47093086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:18:47.322524248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:18:55.328705733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:19:02.332210998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:40392"
ts=2026-05-07T12:19:14.403266604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:49742"
ts=2026-05-07T12:19:17.325319307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:19:19.032556355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36522"
ts=2026-05-07T12:19:44.40487768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46192"
ts=2026-05-07T12:19:47.327237635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:19:49.034170527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57712"
level=debug ts=2026-05-07T12:19:53.362674245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:19:53.364227662Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:20:14.406210083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42874"
ts=2026-05-07T12:20:17.328985077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:20:43.318579172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:20:47.330727006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:21:02.344492375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:33076"
ts=2026-05-07T12:21:17.332481258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:21:36.15217257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:21:47.335336882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:22:02.35103471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:37668"
ts=2026-05-07T12:22:17.337228598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:22:19.047500648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49088"
level=error ts=2026-05-07T12:22:23.574105613Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:22:23.575778002Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:22:23.577313568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:22:23.57733637Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T12:22:37.646738366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:22:47.337985902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:23:14.420232882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:59640"
ts=2026-05-07T12:23:17.339551185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:23:19.038225766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:23:19.051907568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44210"
ts=2026-05-07T12:23:47.340628221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:23:49.053881003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50726"
ts=2026-05-07T12:24:02.363183051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47008"
level=debug ts=2026-05-07T12:24:09.42674096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:24:09.428394783Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:24:14.423587418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:46392"
ts=2026-05-07T12:24:17.342866159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:24:19.055651467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38062"
ts=2026-05-07T12:24:32.366890242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:51800"
ts=2026-05-07T12:24:47.345575986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:24:49.057700072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41374"
level=debug ts=2026-05-07T12:24:58.652588739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:25:02.370141646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38282"
ts=2026-05-07T12:25:17.347387717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:25:44.431004831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:59662"
ts=2026-05-07T12:25:47.349360974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:25:49.062495811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56796"
level=debug ts=2026-05-07T12:25:55.272638392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:26:17.351917902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:26:47.354641551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:26:48.717627608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:26:49.067588044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53654"
ts=2026-05-07T12:27:14.438995957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33954"
ts=2026-05-07T12:27:17.356646267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:27:19.070066318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50728"
level=debug ts=2026-05-07T12:27:23.388279242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:27:44.440056421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53958"
ts=2026-05-07T12:27:47.358054867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:28:13.951914386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:28:13.953478317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:28:17.360743327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:28:47.363334145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:28:54.250433126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:29:14.444985084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44940"
ts=2026-05-07T12:29:17.3646612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:29:44.44656244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39962"
ts=2026-05-07T12:29:47.366451245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:29:49.08112386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39964"
level=debug ts=2026-05-07T12:29:55.467164581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:30:02.402781493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57056"
ts=2026-05-07T12:30:17.368418526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:30:44.126317276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:30:44.449151244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44798"
ts=2026-05-07T12:30:47.370639273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:30:49.085832348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38432"
ts=2026-05-07T12:31:02.41731324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:50198"
ts=2026-05-07T12:31:14.450198412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:47116"
ts=2026-05-07T12:31:17.372222558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:31:38.915212828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:31:47.373223686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:31:49.09138158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58042"
ts=2026-05-07T12:32:02.423006199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:36696"
ts=2026-05-07T12:32:17.375012846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=error ts=2026-05-07T12:32:23.575195009Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:32:23.576836617Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:32:23.578324205Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:32:23.578343974Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T12:32:27.345976273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:32:27.347582731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:32:32.426151735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39124"
ts=2026-05-07T12:32:44.45729487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36074"
ts=2026-05-07T12:32:47.376897736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:33:11.224058863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:33:14.458951876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34164"
ts=2026-05-07T12:33:17.37991836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:33:47.382329698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:33:55.863389485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:34:17.384572347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:34:19.104219654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48696"
ts=2026-05-07T12:34:32.437915613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48624"
level=debug ts=2026-05-07T12:34:36.886906589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:34:47.387167521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:35:02.442162911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60002"
ts=2026-05-07T12:35:14.471149437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44194"
ts=2026-05-07T12:35:17.389963152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:35:37.947147343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:35:47.391670661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:36:17.393208903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
level=debug ts=2026-05-07T12:36:21.10100456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:36:21.102615477Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:36:44.47687884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:51374"
ts=2026-05-07T12:36:47.394315586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:36:49.116935705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47064"
level=debug ts=2026-05-07T12:36:57.291064089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:37:14.477720119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55698"
ts=2026-05-07T12:37:17.396257645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:37:19.119145627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40576"
level=debug ts=2026-05-07T12:37:42.279252639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:37:47.397507239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:38:02.464476744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:59320"
ts=2026-05-07T12:38:17.399709838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:38:26.365396595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:38:32.467575472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57550"
ts=2026-05-07T12:38:47.402427479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:39:09.492207252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:39:14.486973998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42456"
ts=2026-05-07T12:39:17.404680135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:39:19.126931317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53826"
ts=2026-05-07T12:39:32.473696907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47308"
ts=2026-05-07T12:39:44.488164081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53958"
ts=2026-05-07T12:39:47.406679115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
ts=2026-05-07T12:39:49.128896705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42564"
level=debug ts=2026-05-07T12:39:55.852948191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T12:39:55.854521887Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:40:17.409944776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:40:44.494054751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39004"
ts=2026-05-07T12:40:47.412246122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90cc35c5 10.128.0.71:7946"
level=debug ts=2026-05-07T12:40:50.143266696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:41:02.486960247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60742"
ts=2026-05-07T12:41:17.414443975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:41:44.498971679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:56132"
level=debug ts=2026-05-07T12:41:44.713437296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:41:47.415444146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
ts=2026-05-07T12:42:17.416940689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:42:19.141237984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45684"
level=error ts=2026-05-07T12:42:23.578274787Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:42:23.580025257Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:42:23.582669163Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T12:42:23.582692615Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T12:42:32.495593162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34736"
level=debug ts=2026-05-07T12:42:41.837425453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T12:42:44.503575444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53488"
ts=2026-05-07T12:42:47.417721199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4bm42-232eb51b 10.130.0.50:7946"
ts=2026-05-07T12:43:02.498443097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58024"
ts=2026-05-07T12:43:17.419451936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bjntc-ff7fc771 10.129.0.83:7946"
level=debug ts=2026-05-07T12:43:20.298035424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"