level=info ts=2026-05-04T22:53:58.279768105Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T22:53:58.280069787Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-04T22:53:58.282827268Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T22:53:58.283240394Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:53:58.283331871Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=debug ts=2026-05-04T22:53:58.284428329Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:53:58.284516307Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=info ts=2026-05-04T22:53:58.284632122Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-81c429ae
ts=2026-05-04T22:53:58.284729722Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T22:53:58.284745095Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=debug ts=2026-05-04T22:53:58.284771802Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=info ts=2026-05-04T22:53:58.284946932Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-04T22:53:58.292209704Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T22:53:58.292236631Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:53:58.292251417Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.307372ms
level=error ts=2026-05-04T22:53:58.292265638Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T22:53:58.292276287Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T22:53:58.295533837Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:53:58.295550018Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:53:58.303468678Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-04T22:53:58.303523004Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-04T22:53:58.303534315Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-04T22:53:58.303560462Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-04T22:53:58.303576201Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-04T22:53:58.303643749Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-04T22:53:58.303676301Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-04T22:53:58.303693566Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-04T22:53:58.30389284Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-04T22:53:58.303904129Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-04T22:53:58.303938168Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-04T22:53:58.303980925Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-04T22:53:58.304009382Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-04T22:53:58.304150776Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-04T22:53:58.304162095Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-04T22:53:58.304176402Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-04T22:53:59.305232963Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-04T22:53:59.447491205Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-04T22:53:59.447550652Z caller=loki.go:615 msg="Loki started" startup_time=1.240135475s
ts=2026-05-04T22:53:59.846053155Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:53:59.846083286Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:54:02.210481645Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:54:02.21051184Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T22:54:04.448241897Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-04T22:54:04.448312994Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-04T22:54:07.334636366Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:54:07.334660511Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:54:17.753006593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:54:22.023877854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56644"
ts=2026-05-04T22:54:22.793559706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
ts=2026-05-04T22:54:22.793608031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46790"
ts=2026-05-04T22:54:22.794860446Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-05-04T22:54:22.795011673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
ts=2026-05-04T22:54:22.796003103Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.76:7946: dial tcp 10.129.0.76:7946: connect: connection refused"
level=info ts=2026-05-04T22:54:22.796013723Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=24.503730059s
ts=2026-05-04T22:54:23.771811675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41542"
ts=2026-05-04T22:54:26.741292884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:60764"
ts=2026-05-04T22:54:32.957558139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T22:54:35.92984328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:54:56.743441154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:55:00.855736103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39762"
ts=2026-05-04T22:55:02.959705798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T22:55:07.614339395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57062"
level=debug ts=2026-05-04T22:55:21.220021969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:55:30.857483435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42938"
ts=2026-05-04T22:55:32.961467064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:55:37.615460987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60136"
level=debug ts=2026-05-04T22:55:51.649469176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:55:51.651000935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:56:00.859233764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53068"
ts=2026-05-04T22:56:02.963558733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:56:32.965383849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:56:37.618697389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57392"
level=debug ts=2026-05-04T22:56:39.236901173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:57:00.862711282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:33628"
ts=2026-05-04T22:57:02.966550029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T22:57:30.411380112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:57:32.968184883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T22:58:00.867299394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45178"
ts=2026-05-04T22:58:02.96994928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:58:07.623775525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39864"
level=debug ts=2026-05-04T22:58:28.78985416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:58:29.034249202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39596"
ts=2026-05-04T22:58:32.971700264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:59:02.97370946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T22:59:32.975838441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T22:59:37.628580878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37384"
level=debug ts=2026-05-04T22:59:37.928500859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T23:00:02.977443219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:00:29.039113365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:32894"
ts=2026-05-04T23:00:32.979019631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:00:53.487327259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T23:00:53.48880517Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:00:59.040734658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45134"
ts=2026-05-04T23:01:00.879111266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40406"
ts=2026-05-04T23:01:02.979873612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:01:32.981529308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:01:37.939046622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:01:59.043367023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41764"
ts=2026-05-04T23:02:02.982623416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:02:29.045068918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47000"
ts=2026-05-04T23:02:30.884927059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:55256"
ts=2026-05-04T23:02:32.983911272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:02:33.383213876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:03:02.985224098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:03:30.888973764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53302"
ts=2026-05-04T23:03:32.986660054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:03:33.276684672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:04:02.988526699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=info ts=2026-05-04T23:04:04.448726946Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-04T23:04:04.452160675Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:04:04.454444334Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:04:04.456457406Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:04:04.456481285Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T23:04:04.456494406Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-04T23:04:15.458934939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:04:32.989654071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:04:51.100474342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:04:51.102560931Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:04:59.054891972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37842"
ts=2026-05-04T23:05:02.991044032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:05:32.992719923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:05:37.64801551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51532"
level=debug ts=2026-05-04T23:05:43.958297115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:06:02.994266176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:06:29.061301623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37538"
ts=2026-05-04T23:06:30.901685293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48930"
ts=2026-05-04T23:06:32.996121746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:06:37.022981037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:06:37.651241686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56666"
ts=2026-05-04T23:06:59.06276387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38344"
ts=2026-05-04T23:07:02.997853839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:07:29.064770048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59462"
level=debug ts=2026-05-04T23:07:30.673799788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:07:32.999696706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:07:37.65504381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43982"
ts=2026-05-04T23:08:03.001507743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:08:07.657207047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45160"
level=debug ts=2026-05-04T23:08:29.023001339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:08:33.003060825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:09:00.911989893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39484"
ts=2026-05-04T23:09:03.005378183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:09:04.107616242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:09:04.109256131Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:09:07.660427142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52494"
ts=2026-05-04T23:09:33.008112318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:09:44.556364638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:09:59.073831652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49392"
ts=2026-05-04T23:10:03.009822867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:10:07.663858111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56906"
level=debug ts=2026-05-04T23:10:27.67870469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:10:30.918725395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:34958"
ts=2026-05-04T23:10:33.011560437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:10:37.665360373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60418"
ts=2026-05-04T23:11:00.920783656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54070"
ts=2026-05-04T23:11:03.013002203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:11:21.561968397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:11:29.077998683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57462"
ts=2026-05-04T23:11:33.014576813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:12:03.015920734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:12:14.784226576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:12:29.081788835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34494"
ts=2026-05-04T23:12:33.018031394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:13:03.019581457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:13:07.673849621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56098"
level=debug ts=2026-05-04T23:13:13.455927473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:13:13.457883296Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:13:33.021145405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:13:59.087129971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52780"
ts=2026-05-04T23:14:03.023105117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=error ts=2026-05-04T23:14:04.459938323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:14:04.462998325Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:14:04.465343566Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:14:04.465390584Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:14:05.773849523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:14:29.088819546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33842"
ts=2026-05-04T23:14:33.025394915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:14:54.789343412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:14:59.090386811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51720"
ts=2026-05-04T23:15:00.938809716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56204"
ts=2026-05-04T23:15:03.026785382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:15:07.679835532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54282"
ts=2026-05-04T23:15:33.028123437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:15:56.30093055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:16:03.029437943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:16:07.682867564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35514"
ts=2026-05-04T23:16:33.031106718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:16:37.684035906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43508"
level=debug ts=2026-05-04T23:16:49.402315822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:17:00.947243965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46798"
ts=2026-05-04T23:17:03.032907221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:17:07.685460587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49044"
level=debug ts=2026-05-04T23:17:29.331420325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:17:29.333782031Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:17:33.034756621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:17:37.689673198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40464"
ts=2026-05-04T23:18:00.95400684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:38976"
ts=2026-05-04T23:18:03.036669438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:18:04.528245295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:18:33.038088067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:18:37.692673307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56690"
ts=2026-05-04T23:19:03.039962542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:19:06.207414959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:19:33.041638535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:19:53.358094119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:20:00.961770917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:57298"
ts=2026-05-04T23:20:03.042713187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:20:30.963709842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46848"
ts=2026-05-04T23:20:33.04470762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:20:37.801528581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:20:59.112178966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35708"
ts=2026-05-04T23:21:00.965940656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40864"
ts=2026-05-04T23:21:03.046061711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:21:29.113287046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42824"
ts=2026-05-04T23:21:33.047785696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:21:35.118751689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:21:35.121155197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:21:37.702371165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56676"
ts=2026-05-04T23:22:00.969964874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35776"
ts=2026-05-04T23:22:03.048866138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:22:12.2952807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:22:29.116676636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53366"
ts=2026-05-04T23:22:33.05085535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:23:03.052147275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:23:08.862545569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:23:30.975946134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45052"
ts=2026-05-04T23:23:33.05365045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:23:37.709559789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54062"
level=debug ts=2026-05-04T23:23:46.988017392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:24:03.055649245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=error ts=2026-05-04T23:24:04.46084868Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:24:04.462763033Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:24:04.464498947Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:24:04.464530534Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:24:29.122174399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58360"
ts=2026-05-04T23:24:33.057020157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:24:37.712917281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54992"
level=debug ts=2026-05-04T23:24:37.936356879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:25:03.058705346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:25:29.125705913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57800"
ts=2026-05-04T23:25:30.984498635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47492"
ts=2026-05-04T23:25:33.060178597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:25:34.45896651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:25:34.46079747Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:25:37.716358061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41504"
ts=2026-05-04T23:25:59.127192674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54284"
ts=2026-05-04T23:26:03.062233617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:26:07.717420946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42912"
level=debug ts=2026-05-04T23:26:28.68509446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:26:33.064650466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:26:37.719115997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52800"
ts=2026-05-04T23:26:59.129629433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42328"
ts=2026-05-04T23:27:00.98999987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52142"
ts=2026-05-04T23:27:03.066235662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:27:16.096692113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:27:33.068068119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:27:37.722486252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50648"
ts=2026-05-04T23:28:00.994628819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:38878"
level=debug ts=2026-05-04T23:28:02.782302361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:28:03.07071444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:28:07.724078396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47696"
ts=2026-05-04T23:28:33.075489986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:28:56.094120861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:29:03.076752556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:29:33.077907793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:29:50.124641049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:29:50.126255131Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:29:59.145714277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38540"
ts=2026-05-04T23:30:01.002288068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49648"
ts=2026-05-04T23:30:03.079881636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:30:07.728431365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57884"
level=debug ts=2026-05-04T23:30:25.928151164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:30:31.004495279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58294"
ts=2026-05-04T23:30:33.082344126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:31:03.083298869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:31:07.731552144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58380"
level=debug ts=2026-05-04T23:31:07.995874228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:31:33.084618328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:31:52.614904932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:31:59.154891647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45914"
ts=2026-05-04T23:32:03.085570322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:32:33.087531843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:32:35.242931688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:33:01.01469932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59158"
ts=2026-05-04T23:33:03.089242961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:33:07.738711319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44820"
level=debug ts=2026-05-04T23:33:32.09406823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:33:32.09566481Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:33:33.090758671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:34:03.091796613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=error ts=2026-05-04T23:34:04.462296301Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:34:04.464474866Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:34:04.466293101Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:34:04.466313839Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:34:11.083347129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:34:29.163297445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55382"
ts=2026-05-04T23:34:33.093444484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:35:01.023673175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:34346"
ts=2026-05-04T23:35:03.095078417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:35:07.745426668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52318"
level=debug ts=2026-05-04T23:35:09.116163818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:35:33.096453464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:35:47.658557168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:36:03.098405167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:36:29.171942989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43150"
ts=2026-05-04T23:36:33.09991023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:36:37.750117536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52730"
level=debug ts=2026-05-04T23:36:45.082215497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:37:03.101665129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:37:33.103355103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:37:41.90100561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:37:41.902578378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:38:01.038294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:44200"
ts=2026-05-04T23:38:03.10483031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:38:25.692571828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:38:29.179744152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53898"
ts=2026-05-04T23:38:33.105863791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:38:37.756205083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47000"
ts=2026-05-04T23:39:03.108093541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:39:07.757678123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38818"
level=debug ts=2026-05-04T23:39:26.48611602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:39:33.109426845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:39:37.75929843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57840"
ts=2026-05-04T23:39:59.184439614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39148"
ts=2026-05-04T23:40:03.111246523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:40:28.51176497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:40:33.112930893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:40:59.188991949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33706"
ts=2026-05-04T23:41:03.115432249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:41:08.164549264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:41:33.117944628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:41:37.764963016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56190"
ts=2026-05-04T23:42:01.055867841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52622"
ts=2026-05-04T23:42:03.119414589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:42:03.40604596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:42:03.40770644Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:42:33.120987271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:42:37.768445776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57564"
level=debug ts=2026-05-04T23:42:50.960440624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:42:59.197658481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55094"
ts=2026-05-04T23:43:01.059352954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45106"
ts=2026-05-04T23:43:03.122538432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:43:32.881105199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:43:33.12463702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:43:37.772442569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52078"
ts=2026-05-04T23:44:01.063818068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41322"
ts=2026-05-04T23:44:03.126028793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=error ts=2026-05-04T23:44:04.460364434Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:44:04.46245181Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:44:04.464197356Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:44:04.464218661Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:44:10.993113077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:44:33.127420542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:45:03.128983393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:45:06.839616373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:45:07.776961812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58764"
ts=2026-05-04T23:45:33.131441066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:45:37.778432619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35070"
level=debug ts=2026-05-04T23:45:43.171655602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:45:43.173508033Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:45:59.209008182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44642"
ts=2026-05-04T23:46:01.074226053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58554"
ts=2026-05-04T23:46:03.133027476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:46:25.420789687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:31.076136328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:32840"
ts=2026-05-04T23:46:33.134745546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:46:37.781966879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49082"
ts=2026-05-04T23:47:03.136552244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-04T23:47:18.52878762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:47:33.138659531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:48:03.140658436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:48:14.344617434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:48:33.141770574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:49:03.142925618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:49:09.180073593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:49:29.222032158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48228"
ts=2026-05-04T23:49:33.144354434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:49:37.791397721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50404"
level=debug ts=2026-05-04T23:49:54.266020029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:49:54.267620355Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:49:59.22323938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41058"
ts=2026-05-04T23:50:01.090857021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43202"
ts=2026-05-04T23:50:03.146407096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:50:07.7928559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41368"
ts=2026-05-04T23:50:31.093381202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45110"
ts=2026-05-04T23:50:33.148026624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:50:37.794104481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52272"
level=debug ts=2026-05-04T23:50:49.583413703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:51:03.149850259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:51:24.224277344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:51:33.151484204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:51:37.797259886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59934"
ts=2026-05-04T23:52:01.100894317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:38972"
ts=2026-05-04T23:52:03.153200217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:52:22.357115523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:52:33.154256288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:53:03.155918643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:53:22.92119974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:53:29.236164818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51636"
ts=2026-05-04T23:53:31.107539757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50116"
ts=2026-05-04T23:53:33.158215879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:53:59.23750437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50412"
ts=2026-05-04T23:54:01.109291849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56900"
level=debug ts=2026-05-04T23:54:02.370520399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:54:02.37254427Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:54:03.160022867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=error ts=2026-05-04T23:54:04.459785143Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:54:04.461739267Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:54:04.463710725Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:54:04.463732146Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:54:31.111342879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:38762"
ts=2026-05-04T23:54:33.161195716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:54:50.85025473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:55:03.167952939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:55:07.807740613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47152"
ts=2026-05-04T23:55:33.16990403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-04T23:55:34.839660102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:56:03.171806083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:56:07.811578053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35664"
ts=2026-05-04T23:56:29.244875096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50946"
ts=2026-05-04T23:56:31.120229802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47892"
level=debug ts=2026-05-04T23:56:31.605991565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:56:33.173489659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-04T23:56:37.813229071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51736"
ts=2026-05-04T23:57:01.122458501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39170"
ts=2026-05-04T23:57:03.174785571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:57:07.815039813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34090"
level=debug ts=2026-05-04T23:57:28.364463652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:57:33.176889563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-04T23:57:37.815850913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49696"
ts=2026-05-04T23:58:01.126269093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53686"
ts=2026-05-04T23:58:03.177980999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:58:07.817445032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57294"
level=debug ts=2026-05-04T23:58:23.855213043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:58:23.857731813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:58:29.251127761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51328"
ts=2026-05-04T23:58:31.128223328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53148"
ts=2026-05-04T23:58:33.178963757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:58:37.819124651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37114"
ts=2026-05-04T23:58:59.25210239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41924"
ts=2026-05-04T23:59:03.180380954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-04T23:59:06.353315123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:59:29.252924356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49410"
ts=2026-05-04T23:59:33.182047066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-04T23:59:37.821936439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58650"
level=debug ts=2026-05-04T23:59:47.294035858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:00:01.133735375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52016"
ts=2026-05-05T00:00:03.184036478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:00:22.763234895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:00:29.255508096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41224"
ts=2026-05-05T00:00:33.185633649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:01:03.187454617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:01:22.610380807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:01:29.258675366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49900"
ts=2026-05-05T00:01:31.139977463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51516"
ts=2026-05-05T00:01:33.189345455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:01:59.259915895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48420"
ts=2026-05-05T00:02:03.191260831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:02:07.831257238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55788"
level=debug ts=2026-05-05T00:02:18.537851235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:02:18.539457807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:02:29.261249195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42460"
ts=2026-05-05T00:02:33.1923644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:02:59.26326294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49006"
ts=2026-05-05T00:03:03.19410343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:03:07.834204339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46500"
level=debug ts=2026-05-05T00:03:10.394451888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:03:33.195583491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:03:37.835696184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44392"
ts=2026-05-05T00:03:59.267158619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55950"
ts=2026-05-05T00:04:03.196541657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:04:03.474276634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:04:04.459296595Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:04:04.461156774Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:04:04.462937123Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:04:04.462966126Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:04:07.836874511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44216"
ts=2026-05-05T00:04:29.268969964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41938"
ts=2026-05-05T00:04:33.197862567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:04:57.687194414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:05:03.200019258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:05:07.840086377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50798"
ts=2026-05-05T00:05:33.201470015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:05:59.730132415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:06:03.203200373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:06:33.204787452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:06:37.736718683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:06:37.738679675Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:06:37.844798008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43214"
ts=2026-05-05T00:07:03.20656463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:07:26.105216894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:07:33.20832804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:08:03.209864161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:08:07.849427268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34542"
level=debug ts=2026-05-05T00:08:26.256458993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:08:33.211126477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:09:03.213013188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:09:07.853312157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48846"
level=debug ts=2026-05-05T00:09:12.092361936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:09:33.214352124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:09:56.428537119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:10:01.176320327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51696"
ts=2026-05-05T00:10:03.21595661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:10:07.856790925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:32910"
ts=2026-05-05T00:10:33.218398823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:10:37.858205248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45542"
level=debug ts=2026-05-05T00:10:40.713013215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:10:40.714529099Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:10:59.293161841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48522"
ts=2026-05-05T00:11:01.180583433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52256"
ts=2026-05-05T00:11:03.220066733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:11:33.222250721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:11:36.683873387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:11:37.86185956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47842"
ts=2026-05-05T00:11:59.29556574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38774"
ts=2026-05-05T00:12:01.185218839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53100"
ts=2026-05-05T00:12:03.223232184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:12:29.296450155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34270"
level=debug ts=2026-05-05T00:12:32.171067257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:12:33.22554879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:13:01.190138615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42564"
ts=2026-05-05T00:13:03.227841068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:13:07.866472916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35916"
level=debug ts=2026-05-05T00:13:19.992261786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:13:33.229087696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:13:59.301042435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59618"
ts=2026-05-05T00:14:03.23000783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=error ts=2026-05-05T00:14:04.460846244Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:14:04.462686589Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:14:04.464273011Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:14:04.464295042Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:14:07.87090726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33780"
level=debug ts=2026-05-05T00:14:12.599014744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:14:29.302751057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41968"
ts=2026-05-05T00:14:31.196552723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36552"
ts=2026-05-05T00:14:33.231174048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:14:37.872227292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50446"
ts=2026-05-05T00:14:59.304968098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60488"
level=debug ts=2026-05-05T00:15:01.172137391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:15:01.173848756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:15:03.232693883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:15:07.873868896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47014"
ts=2026-05-05T00:15:29.30830678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43092"
ts=2026-05-05T00:15:31.201778757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41960"
ts=2026-05-05T00:15:33.234689436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:15:37.874843703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53260"
level=debug ts=2026-05-05T00:15:38.199670933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:15:59.309787593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52874"
ts=2026-05-05T00:16:03.236541359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:16:31.103429393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:16:31.205754799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50240"
ts=2026-05-05T00:16:33.238174852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:16:37.877749319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58868"
ts=2026-05-05T00:16:59.313165102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39834"
ts=2026-05-05T00:17:03.241660346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:17:07.878555859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46018"
level=debug ts=2026-05-05T00:17:10.127639033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:17:33.243698176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:17:59.996109049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:18:01.212713381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56640"
ts=2026-05-05T00:18:03.245336498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:18:33.246159567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:18:57.394954928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:18:57.396653569Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:19:01.217057301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:32768"
ts=2026-05-05T00:19:03.248062024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:19:07.885704503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58820"
ts=2026-05-05T00:19:33.250164845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:19:36.569369141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:20:03.251721405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:20:17.136866798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:20:29.326122501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53152"
ts=2026-05-05T00:20:31.223763365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:60098"
ts=2026-05-05T00:20:33.252860751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:21:01.225825692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35876"
ts=2026-05-05T00:21:03.254879716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:21:07.892965355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34904"
level=debug ts=2026-05-05T00:21:18.854183476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:21:33.256782493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:22:03.258618816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:22:07.896922403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50994"
level=debug ts=2026-05-05T00:22:11.704125163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:22:31.232203137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54800"
ts=2026-05-05T00:22:33.260315074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:22:59.333520867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43524"
ts=2026-05-05T00:23:03.261783072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:23:07.875747757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:23:07.877516523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:23:31.236692577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37014"
ts=2026-05-05T00:23:33.262738902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:23:37.904804002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33232"
ts=2026-05-05T00:24:01.23839748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49420"
ts=2026-05-05T00:24:03.264722382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=error ts=2026-05-05T00:24:04.460512149Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:24:04.46256059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:24:04.464519784Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:24:04.464542316Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:24:07.906297736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37894"
level=debug ts=2026-05-05T00:24:09.23455721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:24:31.240309036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43130"
ts=2026-05-05T00:24:33.267425129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:24:37.907990827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49592"
ts=2026-05-05T00:24:59.340134158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39964"
ts=2026-05-05T00:25:03.269416456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:25:09.159644375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:25:33.271240895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
level=debug ts=2026-05-05T00:25:59.326805273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:26:03.273382217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:26:07.91215539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45004"
ts=2026-05-05T00:26:33.275302332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:26:37.913861593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45106"
level=debug ts=2026-05-05T00:26:57.165523321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:27:03.277033564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"
ts=2026-05-05T00:27:07.915698618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50424"
ts=2026-05-05T00:27:31.253203949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:44692"
ts=2026-05-05T00:27:33.278263054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
level=debug ts=2026-05-05T00:27:35.537688712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:27:35.53935013Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:27:59.352027915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40546"
ts=2026-05-05T00:28:01.254906624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41644"
ts=2026-05-05T00:28:03.280014631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
level=debug ts=2026-05-05T00:28:26.050228119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:28:31.256677253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54904"
ts=2026-05-05T00:28:33.281661349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:28:37.920949056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48910"
ts=2026-05-05T00:28:59.355062682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41884"
ts=2026-05-05T00:29:03.283732324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vp9mq-44ae33aa 10.129.0.73:7946"
ts=2026-05-05T00:29:07.92213727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41648"
level=debug ts=2026-05-05T00:29:11.682629416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:29:33.285262408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dcfd0b7e 10.129.0.78:7946"
ts=2026-05-05T00:29:37.923365426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46346"
level=debug ts=2026-05-05T00:29:52.88173272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:30:03.286841286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7kclh-103786a7 10.130.0.78:7946"