level=info ts=2026-02-16T13:54:01.736500503Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T13:54:01.736538218Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T13:54:01.738692563Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T13:54:01.739141915Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:54:01.739252192Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.60 inf=eth0
level=debug ts=2026-02-16T13:54:01.739900231Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:54:01.739992099Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.60 inf=eth0
level=info ts=2026-02-16T13:54:01.740449382Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f69d88a4
ts=2026-02-16T13:54:01.740554696Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T13:54:01.740574123Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.60 advertisePort=7946
level=debug ts=2026-02-16T13:54:01.740597775Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.60 advertisePort=7946
level=info ts=2026-02-16T13:54:01.74078919Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T13:54:01.749890532Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T13:54:01.749928227Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:01.749947228Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.161311ms
level=error ts=2026-02-16T13:54:01.749957417Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T13:54:01.749972081Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T13:54:01.753736554Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:01.753758593Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T13:54:01.75813937Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T13:54:01.758141306Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T13:54:01.758166988Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-02-16T13:54:01.758179379Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T13:54:01.758194791Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T13:54:01.758211982Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T13:54:01.758227202Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.758474332Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-02-16T13:54:01.758488562Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-02-16T13:54:01.758494046Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T13:54:01.75854982Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T13:54:01.758606016Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T13:54:01.758640636Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T13:54:01.758851174Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T13:54:01.758884688Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T13:54:01.758904589Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T13:54:02.759057331Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T13:54:02.93772029Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T13:54:02.937771952Z caller=loki.go:599 msg="Loki started" startup_time=1.266060268s
ts=2026-02-16T13:54:03.123238073Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:03.12326648Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:05.867227934Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:05.867259318Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T13:54:07.938443874Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T13:54:07.938499893Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T13:54:13.838538052Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:13.838563423Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:54:17.521925893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:21.91798177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:34016"
ts=2026-02-16T13:54:22.011484581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41570"
ts=2026-02-16T13:54:22.026206847Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused"
ts=2026-02-16T13:54:22.02633378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.60:7946"
ts=2026-02-16T13:54:22.026351823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57576"
ts=2026-02-16T13:54:22.027285584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.58:7946"
ts=2026-02-16T13:54:22.028996153Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
level=info ts=2026-02-16T13:54:22.029011025Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.279033224s
ts=2026-02-16T13:54:27.31165963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54240"
level=debug ts=2026-02-16T13:54:39.760691985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:51.035860158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50094"
ts=2026-02-16T13:54:53.76453547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:54:59.578581129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:55:21.037658925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59224"
ts=2026-02-16T13:55:23.766020907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T13:55:24.837794167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:55:25.79626845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:40144"
ts=2026-02-16T13:55:51.040211645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45498"
level=debug ts=2026-02-16T13:55:51.481301383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:55:51.483242761Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T13:55:53.768146051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T13:55:55.797392213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46038"
ts=2026-02-16T13:55:57.23869446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56180"
ts=2026-02-16T13:56:23.769444718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T13:56:38.94369162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:56:53.770863519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T13:56:57.242291174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38426"
ts=2026-02-16T13:57:23.772133795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T13:57:27.243872862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50084"
level=debug ts=2026-02-16T13:57:46.560482986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:57:51.048801727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51418"
ts=2026-02-16T13:57:53.773796218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T13:58:23.775449764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T13:58:43.704677647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:58:53.777569404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T13:58:57.249042905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39090"
ts=2026-02-16T13:59:23.779821539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T13:59:50.778880105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:59:53.780994506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:00:23.782550573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:00:41.269918879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:00:41.271511049Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:00:53.784001221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:01:23.7854717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:01:30.341899259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:01:51.067221782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37118"
ts=2026-02-16T14:01:53.787167146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:02:21.06926758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38252"
level=debug ts=2026-02-16T14:02:22.983281352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:02:23.788669573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:02:27.258746867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59742"
ts=2026-02-16T14:02:53.789906889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:03:19.809518985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:03:23.792169719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:03:27.261546637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:32900"
ts=2026-02-16T14:03:53.793512116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=info ts=2026-02-16T14:04:07.938790943Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T14:04:07.942545892Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:04:07.94486731Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:04:07.946932088Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:04:07.946991449Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:04:07.947003985Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-02-16T14:04:10.500194991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:04:23.795068469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:04:51.081234974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42122"
ts=2026-02-16T14:04:53.797551774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:04:55.827866638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:34332"
level=debug ts=2026-02-16T14:05:00.687193882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:05:00.689030406Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:05:23.799212396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:05:25.829436147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57326"
ts=2026-02-16T14:05:27.268388294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41392"
level=debug ts=2026-02-16T14:05:50.346596764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:05:51.085710466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39296"
ts=2026-02-16T14:05:53.801528281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:05:55.830774326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:52494"
ts=2026-02-16T14:05:57.269511278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40738"
ts=2026-02-16T14:06:23.803224756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:06:27.271312877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50464"
level=debug ts=2026-02-16T14:06:27.76268875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:06:53.80527353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:07:19.309711043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:07:23.806943844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:07:25.835028288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:45298"
ts=2026-02-16T14:07:53.809061765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:08:01.495506035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:08:23.812240682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:08:27.278038968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37498"
level=debug ts=2026-02-16T14:08:37.152031491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:08:37.154371344Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:08:53.814753127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:08:57.279821311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55226"
ts=2026-02-16T14:09:23.816364359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:09:24.649166068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:09:27.281396064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43332"
ts=2026-02-16T14:09:53.81774065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:10:21.117612414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44634"
level=debug ts=2026-02-16T14:10:21.197817119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:10:23.819387357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:10:27.284444282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60982"
ts=2026-02-16T14:10:51.119579061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38312"
ts=2026-02-16T14:10:53.821118879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:11:04.519440775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:11:21.121483118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58644"
ts=2026-02-16T14:11:23.822965897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:11:53.824716552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:12:04.51250543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:12:23.826712864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:12:25.852968864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46216"
ts=2026-02-16T14:12:27.298429706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49256"
ts=2026-02-16T14:12:53.827952591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:12:55.595741892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:12:55.597236893Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:12:55.854511343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:42184"
ts=2026-02-16T14:13:23.829396281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:13:25.855560082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:40558"
ts=2026-02-16T14:13:27.301299225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48618"
level=debug ts=2026-02-16T14:13:35.229325011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:13:53.830719588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=error ts=2026-02-16T14:14:07.952224353Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:14:07.954298233Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:14:07.956204995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:14:07.956240151Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:14:23.832273401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:14:27.304912043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45930"
level=debug ts=2026-02-16T14:14:28.588778282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:14:53.834296564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:14:55.860935922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:41814"
level=debug ts=2026-02-16T14:15:03.234558638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:15:23.836609285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:15:53.838619053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:15:55.009642973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:15:55.864343207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:37392"
ts=2026-02-16T14:15:57.309801817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50176"
ts=2026-02-16T14:16:23.840818314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:16:25.865853744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:56422"
ts=2026-02-16T14:16:27.311634109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39300"
level=debug ts=2026-02-16T14:16:47.26242789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:16:47.264339612Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:16:51.145270137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34296"
ts=2026-02-16T14:16:53.843049695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:16:57.313445104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33196"
level=debug ts=2026-02-16T14:17:23.054186288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:17:23.844680846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:17:25.868986189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:52922"
ts=2026-02-16T14:17:51.15007871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34224"
ts=2026-02-16T14:17:53.84674971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:17:55.870643637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:43508"
ts=2026-02-16T14:17:57.317708231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44924"
level=debug ts=2026-02-16T14:18:02.076393327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:18:23.848442847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:18:36.725279307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:18:53.849745292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:18:55.873906054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:56952"
ts=2026-02-16T14:19:23.851535985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:19:25.875378894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57720"
level=debug ts=2026-02-16T14:19:39.010379624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:19:53.853162908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:19:55.876240747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:48220"
level=debug ts=2026-02-16T14:20:19.791191344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:20:19.793145987Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:20:23.855338334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:20:25.877543106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:60298"
ts=2026-02-16T14:20:51.163014021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41552"
ts=2026-02-16T14:20:53.856987007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:20:55.878990536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:36376"
ts=2026-02-16T14:20:57.327473128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53434"
level=debug ts=2026-02-16T14:21:04.178434451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:21:21.165558726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48742"
ts=2026-02-16T14:21:23.859281141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:21:51.167304395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46918"
ts=2026-02-16T14:21:53.861446583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:21:58.730379399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:22:23.863032533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:22:36.089550285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:22:53.864647038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:22:57.334593913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45746"
level=debug ts=2026-02-16T14:23:18.133881338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:23:23.86639704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:23:27.336204843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46176"
ts=2026-02-16T14:23:51.176554445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38758"
ts=2026-02-16T14:23:53.867921484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=error ts=2026-02-16T14:24:07.951196976Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:24:07.953329693Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:24:07.955328196Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:24:07.955363676Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:24:11.530963743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:24:11.533102033Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:24:21.179329146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58380"
ts=2026-02-16T14:24:23.86892235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:24:27.339776159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58328"
ts=2026-02-16T14:24:53.8703182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:24:55.892314195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57450"
ts=2026-02-16T14:24:57.341099735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56958"
level=debug ts=2026-02-16T14:25:06.420597591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:25:23.872338855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:25:44.087320565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:25:53.873920664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:25:55.895664761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:34254"
ts=2026-02-16T14:26:23.87545207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:26:27.347465152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55450"
level=debug ts=2026-02-16T14:26:39.454794619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:26:53.877099797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:27:20.17470753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:27:23.878306071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:27:25.901274392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46532"
ts=2026-02-16T14:27:27.350979605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:54342"
ts=2026-02-16T14:27:53.879423937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:28:18.773860721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:28:18.776214769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:28:21.196112244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41916"
ts=2026-02-16T14:28:23.881374206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:28:25.90471153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:44316"
ts=2026-02-16T14:28:53.882842505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:28:57.355851527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49848"
level=debug ts=2026-02-16T14:29:20.395751534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:29:21.201360873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:32976"
ts=2026-02-16T14:29:23.88390692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:29:51.203875891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57828"
ts=2026-02-16T14:29:53.886040149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:29:57.359857516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56302"
level=debug ts=2026-02-16T14:30:05.353111454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:30:21.209855089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54286"
ts=2026-02-16T14:30:23.887895457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:30:25.910866436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:32834"
ts=2026-02-16T14:30:51.211881957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55044"
level=debug ts=2026-02-16T14:30:52.738815365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:30:53.890881609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:30:57.36186851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56988"
ts=2026-02-16T14:31:21.213887969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48438"
ts=2026-02-16T14:31:23.894200896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:31:27.362885745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:54050"
level=debug ts=2026-02-16T14:31:30.556424559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:31:53.895688989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:31:57.364677074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35874"
level=debug ts=2026-02-16T14:32:19.357491937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:32:19.359391382Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:32:23.897435151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:32:25.91637782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46726"
ts=2026-02-16T14:32:27.366562125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41614"
ts=2026-02-16T14:32:51.220993396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50088"
ts=2026-02-16T14:32:53.898824777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:33:00.116994709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:33:23.900025541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:33:53.901342385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:33:54.582443088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:34:07.95123522Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:34:07.953531771Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:34:07.956038323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:34:07.956074698Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:34:23.902501587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:34:29.309217915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:53.904230939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:34:57.374499779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37002"
level=debug ts=2026-02-16T14:35:18.437007138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:35:23.905672747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:35:53.907068975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:36:08.931145394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:36:08.932811654Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:36:21.235781333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37066"
ts=2026-02-16T14:36:23.909544987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:36:53.911440578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:37:01.49881292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:37:21.239589002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53018"
ts=2026-02-16T14:37:23.913139662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:37:25.933646782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57596"
ts=2026-02-16T14:37:27.384057003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45586"
level=debug ts=2026-02-16T14:37:47.799571654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:37:51.24108294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54966"
ts=2026-02-16T14:37:53.914735035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:37:55.934891626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:37510"
ts=2026-02-16T14:38:23.916682441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:38:40.697620108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:38:53.917660822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:39:23.919889616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:39:25.940316448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:45366"
ts=2026-02-16T14:39:27.391276039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53984"
level=debug ts=2026-02-16T14:39:33.07189003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:39:53.922504117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:39:55.941449346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:43392"
level=debug ts=2026-02-16T14:40:18.797363347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:40:18.79910493Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:40:23.92398222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:40:53.926222342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:41:16.713039891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:41:23.927661741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:41:53.929130991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:42:06.149507526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:42:21.260393147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49532"
ts=2026-02-16T14:42:23.930580162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:42:51.262719231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51206"
level=debug ts=2026-02-16T14:42:53.594540144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:42:53.932057836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:43:23.934307257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:43:31.481255716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:43:53.936264534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=error ts=2026-02-16T14:44:07.951015741Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:44:07.95285577Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:44:07.954676234Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:44:07.954708076Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:44:21.269466158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50616"
level=debug ts=2026-02-16T14:44:23.203681726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:44:23.205249792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:44:23.937692685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:44:27.411770987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39782"
ts=2026-02-16T14:44:53.939159106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:44:57.413728125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48276"
level=debug ts=2026-02-16T14:45:02.763324469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:45:23.940030636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:45:25.959359603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:38362"
ts=2026-02-16T14:45:27.415255876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59650"
ts=2026-02-16T14:45:51.275868346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57930"
ts=2026-02-16T14:45:53.94105953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:45:54.560169616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:46:21.278306064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56392"
ts=2026-02-16T14:46:23.942113323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:46:25.961772177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:34996"
level=debug ts=2026-02-16T14:46:32.130428942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:46:51.280522637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53664"
ts=2026-02-16T14:46:53.943622481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:47:22.160579721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:47:23.944952474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
ts=2026-02-16T14:47:27.421946356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38812"
ts=2026-02-16T14:47:51.28415983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53468"
ts=2026-02-16T14:47:53.946911916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:48:18.621008091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:48:18.62265679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:48:23.948940659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:48:25.968867037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57774"