level=info ts=2026-04-20T17:14:05.331818853Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T17:14:05.331856059Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T17:14:05.333954656Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T17:14:05.33441742Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T17:14:05.334530482Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.96 inf=eth0
level=debug ts=2026-04-20T17:14:05.335175878Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T17:14:05.335273528Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.96 inf=eth0
level=info ts=2026-04-20T17:14:05.335357379Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-bcde487f
ts=2026-04-20T17:14:05.335446329Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T17:14:05.335466503Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.96 advertisePort=7946
level=debug ts=2026-04-20T17:14:05.33548717Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.96 advertisePort=7946
level=info ts=2026-04-20T17:14:05.335646057Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T17:14:05.340991989Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T17:14:05.341011439Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T17:14:05.34102923Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.386315ms
level=error ts=2026-04-20T17:14:05.341037103Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T17:14:05.341051698Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T17:14:05.344145793Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T17:14:05.344165414Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T17:14:05.353087774Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T17:14:05.353087465Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-20T17:14:05.353077417Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T17:14:05.353157589Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T17:14:05.35315766Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-20T17:14:05.353172995Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T17:14:05.353460777Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-20T17:14:05.35347208Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T17:14:05.353477662Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-20T17:14:05.353481104Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T17:14:05.353527584Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T17:14:05.353563603Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T17:14:05.353584386Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T17:14:05.353723541Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T17:14:05.353729095Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T17:14:05.353738095Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T17:14:06.354116831Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T17:14:06.354194993Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T17:14:06.354245225Z caller=loki.go:599 msg="Loki started" startup_time=1.10045663s
ts=2026-04-20T17:14:06.438037444Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T17:14:06.4380755Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:14:08.938339351Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T17:14:08.93836826Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T17:14:11.354518909Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T17:14:11.354566494Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T17:14:13.848959144Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T17:14:13.848984444Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T17:14:21.241577537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:14:22.195899927Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T17:14:22.195923509Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:14:29.093292056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:39998"
ts=2026-04-20T17:14:34.065748592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:43968"
level=debug ts=2026-04-20T17:14:40.601370558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:14:47.416780045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:14:49.875779781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.48:7946"
ts=2026-04-20T17:14:49.877338579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.93:7946"
ts=2026-04-20T17:14:49.879206593Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.95:7946: dial tcp 10.129.0.95:7946: connect: connection refused"
ts=2026-04-20T17:14:49.879519908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.88:7946"
ts=2026-04-20T17:14:49.880065927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.96:7946"
ts=2026-04-20T17:14:49.880153362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:43334"
ts=2026-04-20T17:14:49.881279719Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.92:7946: dial tcp 10.129.0.92:7946: connect: connection refused"
ts=2026-04-20T17:14:49.882339036Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.94:7946: dial tcp 10.129.0.94:7946: connect: connection refused"
level=info ts=2026-04-20T17:14:49.882351667Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=44.541295363s
ts=2026-04-20T17:14:52.090515447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:44318"
level=debug ts=2026-04-20T17:15:02.906271883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:15:05.275287675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:39346"
ts=2026-04-20T17:15:17.418776307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:15:27.670317074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:15:31.019667767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:54760"
ts=2026-04-20T17:15:33.682620053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:54824"
ts=2026-04-20T17:15:47.42169868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:15:55.761667101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T17:15:55.763351824Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:16:01.021077074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:34942"
ts=2026-04-20T17:16:17.42465177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:16:31.023024392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53972"
level=debug ts=2026-04-20T17:16:39.601588226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:16:47.427578395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:17:03.687953763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:60440"
ts=2026-04-20T17:17:05.287224614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:40952"
ts=2026-04-20T17:17:17.430934679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:17:34.287556401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:17:47.432860253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:18:17.43403993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:18:31.032270796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:39542"
ts=2026-04-20T17:18:35.296343724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34694"
level=debug ts=2026-04-20T17:18:35.659015193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:18:47.43550409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:19:03.696934615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:60342"
ts=2026-04-20T17:19:17.436777833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:19:43.744468739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T17:19:47.439308372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:20:01.039381514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60018"
ts=2026-04-20T17:20:17.442698085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:20:31.040997494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:58078"
level=debug ts=2026-04-20T17:20:45.687985617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T17:20:45.689639631Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:20:47.445400165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:21:17.447505872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:21:26.194926451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:21:31.046183198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47358"
ts=2026-04-20T17:21:33.708170663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:45360"
ts=2026-04-20T17:21:47.448864149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:22:05.315943205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:40446"
level=debug ts=2026-04-20T17:22:07.02361585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:22:17.450151706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:22:33.712685997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:39618"
ts=2026-04-20T17:22:47.451817171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:23:02.675551363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:23:05.321270776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:58710"
ts=2026-04-20T17:23:17.453561412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:23:33.716511313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:48112"
level=debug ts=2026-04-20T17:23:40.527543852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:23:47.455414172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:24:01.058591205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:37600"
ts=2026-04-20T17:24:03.718048891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:43636"
ts=2026-04-20T17:24:05.326844804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50222"
level=info ts=2026-04-20T17:24:11.355513455Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T17:24:11.359225363Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:24:11.361571888Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:24:11.36401477Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:24:11.364081774Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T17:24:11.364100145Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T17:24:17.456511506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:24:38.44527207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:24:38.446984276Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:24:47.457626552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:25:01.061024426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36210"
ts=2026-04-20T17:25:17.460396342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:25:31.046257675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:25:47.463651027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:26:03.724793889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:60572"
ts=2026-04-20T17:26:17.46590072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:26:27.185145099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:26:31.067314476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:46854"
ts=2026-04-20T17:26:47.467696893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:27:03.729713983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:52864"
ts=2026-04-20T17:27:17.469666552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:27:25.068428282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:27:31.070169327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36656"
ts=2026-04-20T17:27:33.730916171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:49278"
ts=2026-04-20T17:27:47.472838277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:28:01.07165237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:43832"
ts=2026-04-20T17:28:05.349948903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55650"
ts=2026-04-20T17:28:17.474698266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:28:21.282408534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:28:33.733661169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:56912"
ts=2026-04-20T17:28:47.476372971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:29:01.076235358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:42786"
ts=2026-04-20T17:29:03.734739614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:40454"
level=debug ts=2026-04-20T17:29:14.000522156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:29:14.002166564Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:29:17.477298954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:29:47.480035712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:30:06.9613123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:30:17.48323036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:30:47.486261733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:30:51.14396699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:31:03.743019471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:56436"
ts=2026-04-20T17:31:17.488545383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:31:33.744448583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:45572"
ts=2026-04-20T17:31:35.371411415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45634"
level=debug ts=2026-04-20T17:31:46.433103421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:31:47.490375984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:32:17.491384821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:32:29.53330029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:32:35.377236199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:40964"
ts=2026-04-20T17:32:47.493291735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:33:01.091357922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:58732"
ts=2026-04-20T17:33:03.749320227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:46872"
ts=2026-04-20T17:33:05.379659083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:39488"
ts=2026-04-20T17:33:17.495214173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:33:19.396887243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:33:19.398814872Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:33:47.497370133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:34:06.066732891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:34:11.367376664Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:34:11.369572492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:34:11.371575674Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:34:11.371612373Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:34:17.49999038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:34:33.755478173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:52124"
ts=2026-04-20T17:34:35.388684703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:47384"
ts=2026-04-20T17:34:47.502466007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:35:01.100410704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:52482"
level=debug ts=2026-04-20T17:35:06.551288194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:35:17.504268239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:35:33.758940352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:55536"
ts=2026-04-20T17:35:47.508474609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:36:03.760258651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:60206"
level=debug ts=2026-04-20T17:36:04.041486881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:36:05.398311227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50520"
ts=2026-04-20T17:36:17.511186469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:36:39.161927555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:36:47.513009739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:37:17.514703226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:37:31.108521424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36584"
ts=2026-04-20T17:37:33.76798026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:55800"
level=debug ts=2026-04-20T17:37:38.800279291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:37:38.801964851Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:37:47.516958274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:38:17.519039664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:38:24.756150538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:38:47.521614855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:39:05.666253546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:39:17.524291604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:39:33.775545598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:40504"
ts=2026-04-20T17:39:35.420206939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34056"
level=debug ts=2026-04-20T17:39:43.868204533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:39:47.526263933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:40:01.120756659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55290"
ts=2026-04-20T17:40:03.776903961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:59146"
ts=2026-04-20T17:40:17.527440762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:40:31.093914306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:40:35.425622013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:54894"
ts=2026-04-20T17:40:47.534296962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:41:03.781862566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:46802"
ts=2026-04-20T17:41:05.428908421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:54442"
level=debug ts=2026-04-20T17:41:12.527814835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:41:12.529805474Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:41:17.53885743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:41:31.1272955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:39582"
ts=2026-04-20T17:41:33.783422643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:41360"
ts=2026-04-20T17:41:47.541341713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:41:51.947869833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:42:17.544736003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:42:32.998472049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:42:35.437900283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55124"
ts=2026-04-20T17:42:47.546498993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:43:01.132335243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55902"
level=debug ts=2026-04-20T17:43:13.752582791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:43:17.548693383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:43:33.790099771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:48234"
ts=2026-04-20T17:43:47.550647881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:43:53.157564157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:44:11.375969263Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:44:11.378256145Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:44:11.38010791Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T17:44:11.380164521Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:44:17.55253865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:44:31.136917754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:40910"
ts=2026-04-20T17:44:35.450153084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:40004"
level=debug ts=2026-04-20T17:44:36.041532039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:44:36.043524385Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:44:47.553765723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:45:03.794641138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:34962"
ts=2026-04-20T17:45:17.555681899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
level=debug ts=2026-04-20T17:45:23.759386727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:45:47.558169601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:46:05.459172874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:51100"
level=debug ts=2026-04-20T17:46:14.445650129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:46:17.560178319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:46:33.798668628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:46698"
ts=2026-04-20T17:46:47.562052249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:47:03.597000469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:47:03.799802821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:45228"
ts=2026-04-20T17:47:05.465550138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:43508"
ts=2026-04-20T17:47:17.564156977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6dlpb-ad29117c 10.129.0.88:7946"
ts=2026-04-20T17:47:35.468222904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:54902"
ts=2026-04-20T17:47:47.566010502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:47:52.487891493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:48:03.803812662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:38100"
ts=2026-04-20T17:48:05.471481583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:57480"
ts=2026-04-20T17:48:17.569087601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
ts=2026-04-20T17:48:47.571559725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22586e03 10.130.0.48:7946"
level=debug ts=2026-04-20T17:48:51.213601228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T17:48:51.215384997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T17:49:01.156587647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:51190"
ts=2026-04-20T17:49:03.807269294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:59376"
ts=2026-04-20T17:49:05.477232262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:53276"
ts=2026-04-20T17:49:17.574024679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:49:30.168172572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:49:31.158339663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60660"
ts=2026-04-20T17:49:33.808670319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:48972"
ts=2026-04-20T17:49:47.575164342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
level=debug ts=2026-04-20T17:50:17.277059226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T17:50:17.576284828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"
ts=2026-04-20T17:50:33.812860356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.93:36130"
ts=2026-04-20T17:50:47.577773082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9b97b-1b632f88 10.129.0.93:7946"