level=info ts=2026-02-16T11:30:59.223271621Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T11:30:59.223322883Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T11:30:59.225114285Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T11:30:59.225626434Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:30:59.225820423Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.54 inf=eth0
level=debug ts=2026-02-16T11:30:59.227008675Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:30:59.227109247Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.54 inf=eth0
level=info ts=2026-02-16T11:30:59.227145287Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-08edc817
ts=2026-02-16T11:30:59.227251375Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T11:30:59.227269216Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.54 advertisePort=7946
level=debug ts=2026-02-16T11:30:59.22729549Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.54 advertisePort=7946
level=info ts=2026-02-16T11:30:59.227470547Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T11:30:59.232724741Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:30:59.23274842Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:30:59.232759375Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.292269ms
level=error ts=2026-02-16T11:30:59.232766951Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T11:30:59.232780282Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:30:59.235582004Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:59.235618941Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:30:59.244430426Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T11:30:59.244461668Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-02-16T11:30:59.244485453Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T11:30:59.244502681Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T11:30:59.24452086Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T11:30:59.244551646Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T11:30:59.244670709Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:59.244696927Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:59.244919816Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-02-16T11:30:59.244932029Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T11:30:59.244989271Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T11:30:59.244992427Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T11:30:59.245018397Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T11:30:59.245205903Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T11:30:59.245216237Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T11:30:59.245231103Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T11:31:00.246369764Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T11:31:00.446973589Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T11:31:00.44703263Z caller=loki.go:599 msg="Loki started" startup_time=1.307157339s
ts=2026-02-16T11:31:00.876920811Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:00.876944437Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:03.953777486Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:03.953807883Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:31:05.44774528Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T11:31:05.44781387Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T11:31:09.786689263Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:09.786731213Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:31:17.47056366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:20.224929347Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:20.224961341Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:31:30.127638906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:39.013879715Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.60:7946: dial tcp 10.129.0.60:7946: connect: connection refused"
ts=2026-02-16T11:31:39.015110121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.61:7946"
ts=2026-02-16T11:31:39.017309303Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.63:7946: dial tcp 10.129.0.63:7946: connect: connection refused"
ts=2026-02-16T11:31:39.017454669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.54:7946"
ts=2026-02-16T11:31:39.01747206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38798"
ts=2026-02-16T11:31:39.018940701Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.53:7946: dial tcp 10.130.0.53:7946: connect: connection refused"
ts=2026-02-16T11:31:39.021155742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.58:7946"
level=info ts=2026-02-16T11:31:39.022526643Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=39.789740202s
ts=2026-02-16T11:31:45.0055164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39806"
ts=2026-02-16T11:31:46.57042869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55564"
level=debug ts=2026-02-16T11:31:50.542703957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:51.121004086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:45882"
ts=2026-02-16T11:31:59.825085726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:40576"
ts=2026-02-16T11:32:06.390455393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:32:11.339691731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:32:17.812266738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55772"
ts=2026-02-16T11:32:36.393979075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:32:38.281546997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:32:38.283529305Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:32:41.170306783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:50692"
ts=2026-02-16T11:33:06.397093879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:33:11.172181586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:34560"
level=debug ts=2026-02-16T11:33:15.729652981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:33:36.399739025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:34:06.402040435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:34:21.999874079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:34:36.404332519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:34:47.828657562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49384"
ts=2026-02-16T11:35:06.405989701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:35:19.603921518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:35:36.407394961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:35:41.182173321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:42656"
ts=2026-02-16T11:35:47.834116074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44714"
ts=2026-02-16T11:35:59.846750286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54796"
ts=2026-02-16T11:36:06.409217575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:36:31.582334775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:36:36.411868197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:36:59.852221774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:48626"
ts=2026-02-16T11:37:06.413517827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:37:27.527026946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:37:27.528635038Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:37:29.854911169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45492"
ts=2026-02-16T11:37:36.415462155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:37:47.84607652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51318"
level=debug ts=2026-02-16T11:38:05.994610302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:38:06.418948432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:38:36.422416566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:38:51.871531477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:39:06.42454647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:39:11.199801313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:51442"
ts=2026-02-16T11:39:36.426612206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:39:45.583668564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:39:59.866494658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54678"
ts=2026-02-16T11:40:06.42852423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:40:11.203922203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:55224"
ts=2026-02-16T11:40:17.862314284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55142"
level=debug ts=2026-02-16T11:40:33.199836693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:40:36.430045931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:40:47.865294977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48432"
ts=2026-02-16T11:40:59.871236151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:46526"
level=info ts=2026-02-16T11:41:05.448274836Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T11:41:05.451808559Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:05.453684592Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:05.455828374Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:05.455862829Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:41:05.45587647Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-02-16T11:41:06.432840238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:41:11.206982655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:55412"
level=debug ts=2026-02-16T11:41:32.86412037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:41:32.865799732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:41:36.434608932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:41:41.208137706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:58114"
ts=2026-02-16T11:41:47.870857698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55112"
ts=2026-02-16T11:42:06.43646822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:42:28.768213073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:42:36.439090161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:43:06.44223521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:43:29.029597522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:43:36.44486114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:43:41.222792061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:38948"
ts=2026-02-16T11:44:06.446742704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:44:10.875704666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:44:11.224221565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:53882"
ts=2026-02-16T11:44:17.887869368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48508"
ts=2026-02-16T11:44:36.450049618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:44:59.891435182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37654"
ts=2026-02-16T11:45:06.452238079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:45:07.417907832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:45:17.893647369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41100"
ts=2026-02-16T11:45:36.454936886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:45:41.238255745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:36576"
level=debug ts=2026-02-16T11:45:57.736752936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:45:57.73862616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:46:06.459119627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:46:36.461625186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:46:37.578654785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:46:47.902563111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46810"
ts=2026-02-16T11:46:59.903114051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:35978"
ts=2026-02-16T11:47:06.46418823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:47:23.327703465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:47:36.466760976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:48:06.468942073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:48:11.683317181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:48:36.471565258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:48:59.912391555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:58086"
ts=2026-02-16T11:49:06.474056991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:49:11.259688248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:53202"
level=debug ts=2026-02-16T11:49:11.905190011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:49:29.914893611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:38968"
ts=2026-02-16T11:49:36.475875587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:50:06.477829738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:50:10.953671162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:50:10.957175174Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:50:17.926159584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33424"
ts=2026-02-16T11:50:29.921897704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:40842"
ts=2026-02-16T11:50:36.480240544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:50:41.272359222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:49374"
level=error ts=2026-02-16T11:51:05.460402673Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:05.462768313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:05.464941166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:05.464978274Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:51:06.482786634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:51:11.273913142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:33516"
level=debug ts=2026-02-16T11:51:13.033349844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:51:36.485909684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:52:06.489246972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T11:52:09.595165755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:52:11.278858609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60876"
ts=2026-02-16T11:52:36.492970099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:52:41.280156219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:41060"
ts=2026-02-16T11:52:47.941966458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48264"
level=debug ts=2026-02-16T11:53:05.381407231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:53:06.496091732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T11:53:11.281180707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:51610"
ts=2026-02-16T11:53:36.498534035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:53:47.94829428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52448"
level=debug ts=2026-02-16T11:53:58.496902859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:54:06.500459024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:54:17.951386661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44248"
ts=2026-02-16T11:54:29.949650146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:56818"
ts=2026-02-16T11:54:36.502838367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:54:40.015913034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:54:40.018396637Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:54:59.951974672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54182"
ts=2026-02-16T11:55:06.505025649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:55:27.984556048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:55:36.5071431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:55:41.293558323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:35582"
ts=2026-02-16T11:55:47.960902159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:60936"
ts=2026-02-16T11:56:06.508926053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:56:11.294772373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:34774"
level=debug ts=2026-02-16T11:56:20.237743744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:56:29.960544693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50950"
ts=2026-02-16T11:56:36.511495854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:57:01.829053472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:39170"
ts=2026-02-16T11:57:01.841685637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:39186"
ts=2026-02-16T11:57:06.513493356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
ts=2026-02-16T11:57:11.227970678Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3f64719b (timeout reached)"
ts=2026-02-16T11:57:14.228091656Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3f64719b has failed, no acks received"
ts=2026-02-16T11:57:17.970622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53446"
level=debug ts=2026-02-16T11:57:21.002579543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:57:29.96693194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54494"
ts=2026-02-16T11:57:32.986813098Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3f64719b as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T11:57:35.427086907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:37894"
ts=2026-02-16T11:57:36.515552454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:57:47.974159074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58732"
ts=2026-02-16T11:57:59.969803374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:44262"
ts=2026-02-16T11:58:05.43049455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47982"
ts=2026-02-16T11:58:06.518109928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
level=debug ts=2026-02-16T11:58:07.581497474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:58:35.432283806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:48454"
ts=2026-02-16T11:58:36.520066119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
level=debug ts=2026-02-16T11:58:53.279652883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:58:53.281588295Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:59:06.527670113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
level=debug ts=2026-02-16T11:59:36.187335982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:59:36.530282341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:00:05.440152375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:57504"
ts=2026-02-16T12:00:06.532699994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:00:23.423008325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:00:36.535449912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:00:47.992501203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54648"
level=debug ts=2026-02-16T12:00:58.816358044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:01:05.458922265Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:05.461088071Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:05.463146413Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:05.463186522Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:01:06.538071347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:01:17.995083993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33698"
level=debug ts=2026-02-16T12:01:36.278985195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:01:36.540499909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:01:47.998269052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39384"
ts=2026-02-16T12:02:00.000443599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:59886"
ts=2026-02-16T12:02:05.45149527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:50904"
ts=2026-02-16T12:02:06.543024719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:02:29.894131593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:02:29.895793234Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:02:36.545692786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:03:00.006129685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39670"
ts=2026-02-16T12:03:05.455024962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47928"
ts=2026-02-16T12:03:06.548751461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:03:15.32453824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:03:18.006912847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50070"
ts=2026-02-16T12:03:36.550704522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
ts=2026-02-16T12:03:48.009545581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52680"
level=debug ts=2026-02-16T12:03:54.080761412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:05.460092415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:43530"
ts=2026-02-16T12:04:06.552641058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:04:35.46162517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:51002"
ts=2026-02-16T12:04:36.554886076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
level=debug ts=2026-02-16T12:04:52.442200602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:05:00.0174706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:42992"
ts=2026-02-16T12:05:05.462722908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60860"
ts=2026-02-16T12:05:06.557140464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:05:35.464646085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:43186"
ts=2026-02-16T12:05:36.560304241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:05:43.904600865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:05:48.022582755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54090"
ts=2026-02-16T12:05:54.169844009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:46390"
ts=2026-02-16T12:05:54.180306136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:46400"
ts=2026-02-16T12:06:01.227957152Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3c0b0408 (timeout reached)"
ts=2026-02-16T12:06:04.228329574Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3c0b0408 has failed, no acks received"
ts=2026-02-16T12:06:06.216128883Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3c0b0408 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:06:06.562653895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:06:18.025836368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46012"
level=debug ts=2026-02-16T12:06:28.568407334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:06:28.570160181Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:06:30.029636996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:35424"
ts=2026-02-16T12:06:36.564487748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
ts=2026-02-16T12:06:46.802243811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:59850"
ts=2026-02-16T12:07:06.567233276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:07:18.033906616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50748"
level=debug ts=2026-02-16T12:07:29.53030183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:07:36.570236115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:08:06.57201587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
level=debug ts=2026-02-16T12:08:12.276684304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:08:36.573426481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
ts=2026-02-16T12:08:46.810320278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:42632"
level=debug ts=2026-02-16T12:08:53.618860646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:09:00.04926892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:51434"
ts=2026-02-16T12:09:06.575923101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:09:18.046113181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51368"
ts=2026-02-16T12:09:36.578895605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:09:54.248147249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:10:06.581760102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:10:30.055683031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:41162"
ts=2026-02-16T12:10:36.584699471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:10:50.947239541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:10:50.949138274Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:05.462981723Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:05.465134763Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:05.468028711Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:05.468076531Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:11:06.587546432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:11:16.824433053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:37856"
ts=2026-02-16T12:11:36.590113193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:11:39.243786595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:12:00.064474261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45198"
ts=2026-02-16T12:12:06.59240571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
level=debug ts=2026-02-16T12:12:14.932493271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:12:36.59511193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:12:46.835410417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60898"
level=debug ts=2026-02-16T12:13:00.490025339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:13:06.597500502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:13:30.07161746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:41054"
ts=2026-02-16T12:13:36.599672656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:13:46.839489621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:41202"
level=debug ts=2026-02-16T12:13:53.148500386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:14:06.602350144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:14:11.22810383Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b4cd34c9 (timeout reached)"
ts=2026-02-16T12:14:14.228520103Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b4cd34c9 has failed, no acks received"
ts=2026-02-16T12:14:18.077656499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53590"
ts=2026-02-16T12:14:30.076680168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:60044"
ts=2026-02-16T12:14:31.216817802Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b4cd34c9 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:14:31.22850213Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b4cd34c9 (timeout reached)"
ts=2026-02-16T12:14:34.228818822Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b4cd34c9 has failed, no acks received"
ts=2026-02-16T12:14:35.149328009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:55978"
ts=2026-02-16T12:14:35.160619391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:55990"
ts=2026-02-16T12:14:36.605199573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:14:38.811435893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:14:38.813043913Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:15:06.607987826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:15:19.44983472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:15:26.508394268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:59706"
ts=2026-02-16T12:15:36.61098708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:16:01.613766736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:16:06.613974223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:16:36.617542381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:16:46.124197551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:17:00.090186193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:40040"
ts=2026-02-16T12:17:06.620608967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:17:26.517353311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:55320"
ts=2026-02-16T12:17:36.6222063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:17:47.053629953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:18:00.094018623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:43658"
ts=2026-02-16T12:18:06.624618647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:18:18.102610394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34126"
level=debug ts=2026-02-16T12:18:21.326878296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:18:21.329021701Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:18:26.521114391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:38984"
ts=2026-02-16T12:18:36.626933218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:19:02.214095768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:19:06.629987516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:19:30.103248948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:36100"
ts=2026-02-16T12:19:36.632133277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:19:38.441777681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:19:48.112443935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:45968"
ts=2026-02-16T12:19:56.528318601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:57740"
ts=2026-02-16T12:20:00.105425394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:41334"
ts=2026-02-16T12:20:06.635016385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:20:30.573843867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:20:36.637149637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=error ts=2026-02-16T12:21:05.460246121Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:05.462344116Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:05.464283022Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:05.464313795Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:21:06.639636952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:21:11.440757678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:21:18.122123276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54704"
ts=2026-02-16T12:21:26.53438074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:51242"
ts=2026-02-16T12:21:36.642071021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:21:48.935374283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:21:48.9369305Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:21:56.5358114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:44290"
ts=2026-02-16T12:22:00.115115836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:40634"
ts=2026-02-16T12:22:06.643673472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:22:26.536912786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:57640"
ts=2026-02-16T12:22:36.646893041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:22:38.709854773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:23:06.648791472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:23:18.135821015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54750"
level=debug ts=2026-02-16T12:23:36.502859687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:23:36.650601995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:23:56.545045952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:43262"
ts=2026-02-16T12:24:00.125908883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:58000"
ts=2026-02-16T12:24:06.653560661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:24:16.228969764Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-929db2e7 (timeout reached)"
level=debug ts=2026-02-16T12:24:16.747891886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:24:19.228326137Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-929db2e7 has failed, no acks received"
ts=2026-02-16T12:24:21.229124679Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-929db2e7 (timeout reached)"
ts=2026-02-16T12:24:24.229182576Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-929db2e7 has failed, no acks received"
ts=2026-02-16T12:24:26.017502279Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-929db2e7 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:24:30.128080772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:58834"
ts=2026-02-16T12:24:36.656042983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:24:43.139215197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:56262"
ts=2026-02-16T12:24:43.149641798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:56278"
level=debug ts=2026-02-16T12:24:59.936244968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:25:00.13074463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:57888"
ts=2026-02-16T12:25:06.658609233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:25:13.142200707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:58486"
ts=2026-02-16T12:25:18.147636423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44054"
ts=2026-02-16T12:25:36.661371147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:25:48.15104016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58716"
level=debug ts=2026-02-16T12:25:55.828949529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:25:55.830678717Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:26:06.663898112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:26:18.154356011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58862"
ts=2026-02-16T12:26:36.666010082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:26:38.585679174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:27:00.140752474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:42704"
ts=2026-02-16T12:27:06.668850212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:27:13.150755447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:33890"
ts=2026-02-16T12:27:18.16062811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53458"
ts=2026-02-16T12:27:36.67200367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
level=debug ts=2026-02-16T12:27:38.210971846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:28:06.674535751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
ts=2026-02-16T12:28:36.676528577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:28:39.045288947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:28:43.157056448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:56548"
ts=2026-02-16T12:29:00.151421959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39250"
ts=2026-02-16T12:29:06.67807601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:29:17.918382601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:29:36.679433388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
ts=2026-02-16T12:30:00.155814288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:56804"
level=debug ts=2026-02-16T12:30:00.33587051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:30:00.337730421Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:30:06.681677704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xzbjr-db1d6c30 10.128.0.58:7946"
ts=2026-02-16T12:30:30.158040856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37040"
ts=2026-02-16T12:30:36.68432983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:30:43.166402789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:52676"