level=info ts=2026-02-16T13:53:02.451959186Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T13:53:02.451991328Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T13:53:02.453049794Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T13:53:02.453462784Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:53:02.453548993Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.66 inf=eth0
level=debug ts=2026-02-16T13:53:02.454590922Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:53:02.454667807Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.66 inf=eth0
level=info ts=2026-02-16T13:53:02.454747273Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-a4df028c
ts=2026-02-16T13:53:02.454824406Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T13:53:02.454835649Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=debug ts=2026-02-16T13:53:02.45485079Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=info ts=2026-02-16T13:53:02.454949227Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T13:53:02.462366938Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T13:53:02.462386547Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:53:02.462399598Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.452119ms
level=error ts=2026-02-16T13:53:02.462410372Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T13:53:02.462425275Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T13:53:02.464676815Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:02.46470964Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:53:02.468972054Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-02-16T13:53:02.469033915Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T13:53:02.469030654Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T13:53:02.469048694Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T13:53:02.469040025Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T13:53:02.469083638Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T13:53:02.469193971Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:53:02.469212721Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T13:53:02.469311991Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-02-16T13:53:02.4693198Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T13:53:02.469369682Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T13:53:02.469369276Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T13:53:02.469391199Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T13:53:02.469555123Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T13:53:02.469568395Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T13:53:02.469585992Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T13:53:03.470219965Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T13:53:03.607558922Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T13:53:03.607627428Z caller=loki.go:599 msg="Loki started" startup_time=1.213622182s
ts=2026-02-16T13:53:04.19471398Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:04.194740168Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:53:07.259780862Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:07.259804678Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T13:53:08.608037661Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T13:53:08.608100177Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T13:53:11.399383825Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:11.399409202Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:53:19.195558663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:53:23.512487702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52922"
ts=2026-02-16T13:53:23.802972793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44870"
ts=2026-02-16T13:53:23.989185573Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T13:53:23.989385899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46124"
ts=2026-02-16T13:53:23.989427197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
level=info ts=2026-02-16T13:53:23.989897601Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.52746579s
ts=2026-02-16T13:53:24.854004319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53456"
level=debug ts=2026-02-16T13:53:37.693305602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:53:46.040714059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T13:53:57.972701743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:05.639073693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48508"
ts=2026-02-16T13:54:16.043860287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T13:54:20.520523194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:38.129501282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39232"
ts=2026-02-16T13:54:46.047096371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T13:54:51.16319309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:54:51.164797409Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T13:55:16.049839434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T13:55:31.584101715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:55:35.645358201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41942"
ts=2026-02-16T13:55:46.052604105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:56:16.055535158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T13:56:17.912402612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43500"
level=debug ts=2026-02-16T13:56:20.933493245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:56:46.058821516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T13:57:12.953385592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:57:16.061744484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:57:46.064411928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:57:47.920401149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42744"
ts=2026-02-16T13:58:16.067058449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T13:58:20.786810874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:58:38.147736012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33116"
ts=2026-02-16T13:58:46.069464152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T13:59:05.658378367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33504"
ts=2026-02-16T13:59:16.072765417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T13:59:19.846124883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:59:19.848008085Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T13:59:46.075957467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:00:08.154963669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34628"
level=debug ts=2026-02-16T14:00:15.340581855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:00:16.078805206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:00:17.931879809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41058"
ts=2026-02-16T14:00:38.158400432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50190"
ts=2026-02-16T14:00:46.081858593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:00:57.072808069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:01:16.08560517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:01:35.667966643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38786"
ts=2026-02-16T14:01:46.088321619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:01:53.787199666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:02:05.671157117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43946"
ts=2026-02-16T14:02:16.091304633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:02:17.941894501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60998"
ts=2026-02-16T14:02:46.094388397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:02:47.945633537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56154"
level=debug ts=2026-02-16T14:02:52.501905637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:03:08.168673739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57490"
level=info ts=2026-02-16T14:03:08.608109001Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T14:03:08.611919643Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:03:08.614138596Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:03:08.616197444Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:03:08.616240503Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:03:08.616254531Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-02-16T14:03:16.097776934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:03:35.67686553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49044"
level=debug ts=2026-02-16T14:03:46.059792207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:03:46.061318723Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:03:46.101536712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:04:08.174318738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59228"
ts=2026-02-16T14:04:16.104476899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:04:17.953902454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:32884"
level=debug ts=2026-02-16T14:04:29.343955387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:04:38.177485779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47068"
ts=2026-02-16T14:04:46.108007274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:04:47.957280932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44950"
level=debug ts=2026-02-16T14:05:05.559719214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:05:05.684142922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56714"
ts=2026-02-16T14:05:16.111264035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:05:38.182162585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59858"
ts=2026-02-16T14:05:46.114121079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:05:47.963062818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45342"
level=debug ts=2026-02-16T14:06:03.300318074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:06:05.691679604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35426"
ts=2026-02-16T14:06:16.117194394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:06:17.966061552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59892"
ts=2026-02-16T14:06:38.186758689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40652"
ts=2026-02-16T14:06:46.12012922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:06:47.969058218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45248"
level=debug ts=2026-02-16T14:06:58.005718023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:07:16.122907568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:07:39.481561403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:07:39.483264608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:07:46.133295705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:08:08.194034215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45984"
ts=2026-02-16T14:08:16.136900059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:08:25.939158859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:08:38.197358667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44926"
ts=2026-02-16T14:08:46.139797415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:09:08.200399965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36638"
level=debug ts=2026-02-16T14:09:08.581390616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:09:16.142876427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:09:38.202875918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50698"
ts=2026-02-16T14:09:46.145345438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:09:50.722341634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:10:08.20566549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33182"
ts=2026-02-16T14:10:16.148139828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:10:35.706770341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40102"
level=debug ts=2026-02-16T14:10:44.181964254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:10:46.15107671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:10:47.988609122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33824"
ts=2026-02-16T14:11:16.154213253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:11:37.745581706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:11:37.7480289Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:11:46.157351785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:11:47.994382452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44194"
ts=2026-02-16T14:12:16.159788607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:12:35.742854296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52272"
ts=2026-02-16T14:12:38.2270231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59544"
level=debug ts=2026-02-16T14:12:39.950030417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:12:46.162555547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:12:48.000476117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60178"
level=error ts=2026-02-16T14:13:08.62183565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:13:08.624527473Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:13:08.627054861Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:13:08.627096642Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:13:16.165991035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:13:27.679470925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:13:46.169597153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:13:48.006889182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51258"
level=debug ts=2026-02-16T14:14:03.317816231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:14:08.233088699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43848"
ts=2026-02-16T14:14:16.173168272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:14:35.752869614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59244"
ts=2026-02-16T14:14:46.175862511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:14:48.013518901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50766"
level=debug ts=2026-02-16T14:14:59.796373459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:15:16.179223097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:15:18.016857457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50896"
ts=2026-02-16T14:15:46.182364549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:15:47.483191433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:15:47.484716157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:16:05.760013278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56812"
ts=2026-02-16T14:16:08.243855945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54216"
ts=2026-02-16T14:16:15.441826844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45804"
ts=2026-02-16T14:16:15.453521761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45810"
ts=2026-02-16T14:16:16.185044914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:16:24.45553741Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-daad38b4 (timeout reached)"
ts=2026-02-16T14:16:27.456383605Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-daad38b4 has failed, no acks received"
ts=2026-02-16T14:16:34.455892104Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-daad38b4 (timeout reached)"
ts=2026-02-16T14:16:35.763133398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37826"
ts=2026-02-16T14:16:37.456112451Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-daad38b4 has failed, no acks received"
level=debug ts=2026-02-16T14:16:44.418040299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:16:46.187583601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:16:47.4569652Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-daad38b4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T14:17:08.248081534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40382"
ts=2026-02-16T14:17:16.190413352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:17:26.990498372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54210"
level=debug ts=2026-02-16T14:17:33.116384419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:17:35.767484225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52492"
ts=2026-02-16T14:17:46.194017974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:18:08.253986753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57672"
ts=2026-02-16T14:18:16.197354019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
level=debug ts=2026-02-16T14:18:21.597452106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:18:46.200861628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:18:56.997547563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49540"
ts=2026-02-16T14:19:05.774573134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38924"
ts=2026-02-16T14:19:08.258281179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59908"
level=debug ts=2026-02-16T14:19:09.900187052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:19:16.204472398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:19:38.261664611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50870"
level=debug ts=2026-02-16T14:19:44.919507805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:19:44.921278054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:19:46.207496681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:20:16.2103823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:20:34.567183417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:20:35.780830291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60774"
ts=2026-02-16T14:20:38.266413863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55798"
ts=2026-02-16T14:20:46.213822734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:21:16.216700016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:21:27.009944059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57022"
level=debug ts=2026-02-16T14:21:36.171670739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:21:38.271035246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58384"
ts=2026-02-16T14:21:46.219648486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:22:08.274064915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55452"
ts=2026-02-16T14:22:16.22241528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:22:18.909887769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:22:27.014900577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55248"
ts=2026-02-16T14:22:46.225667745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:22:53.527615929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:23:08.621050922Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:23:08.623275957Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:23:08.625243142Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:23:08.625277046Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:23:16.22872109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:23:27.019845836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35344"
level=debug ts=2026-02-16T14:23:31.544063145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:23:31.546267308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:23:35.792775575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42472"
ts=2026-02-16T14:23:46.231482632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:24:16.234778698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:24:24.991471148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:24:35.797220302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35146"
ts=2026-02-16T14:24:38.365538994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51702"
ts=2026-02-16T14:24:38.378804631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51718"
ts=2026-02-16T14:24:44.456469802Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1a0c3718 (timeout reached)"
ts=2026-02-16T14:24:46.238177699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:24:47.456913483Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1a0c3718 has failed, no acks received"
ts=2026-02-16T14:24:49.118812645Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1a0c3718 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T14:25:08.285827297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43338"
level=debug ts=2026-02-16T14:25:12.953281623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:25:16.241179233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:25:35.801214723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43962"
ts=2026-02-16T14:25:46.244281263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:25:57.902585683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:26:08.290236163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34128"
ts=2026-02-16T14:26:16.247281669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:26:35.805657463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50502"
ts=2026-02-16T14:26:46.250930621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
level=debug ts=2026-02-16T14:26:57.556518313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:27:05.808176921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49656"
ts=2026-02-16T14:27:16.254402496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:27:46.25730166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
level=debug ts=2026-02-16T14:27:48.595094664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:27:48.596818098Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:28:16.260138531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:28:28.209412694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38818"
level=debug ts=2026-02-16T14:28:36.168362021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:28:46.263049448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:28:58.21305437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52104"
ts=2026-02-16T14:29:05.817182505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37588"
ts=2026-02-16T14:29:16.266263918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:29:17.091669583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:29:38.302678546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56246"
ts=2026-02-16T14:29:46.268917588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:30:12.933801951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:30:16.27229264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:30:38.307898661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43756"
ts=2026-02-16T14:30:46.275028517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:30:58.223295565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58042"
ts=2026-02-16T14:31:05.826914934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41480"
level=debug ts=2026-02-16T14:31:06.887365839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:31:08.310603295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47232"
ts=2026-02-16T14:31:16.277590026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:31:46.280635945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:31:46.608368459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:31:46.610271647Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:31:58.228976023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33304"
ts=2026-02-16T14:32:16.283135884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
ts=2026-02-16T14:32:28.2316323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59654"
level=debug ts=2026-02-16T14:32:33.230924366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:32:35.832578534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54690"
ts=2026-02-16T14:32:46.28577222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=error ts=2026-02-16T14:33:08.620847324Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:33:08.622804243Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:33:08.624550006Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T14:33:08.624584821Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:33:14.387815388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:33:16.288256843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:33:19.455951416Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee30c144 (timeout reached)"
ts=2026-02-16T14:33:22.456068307Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee30c144 has failed, no acks received"
ts=2026-02-16T14:33:29.456377566Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee30c144 (timeout reached)"
ts=2026-02-16T14:33:32.4562871Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee30c144 has failed, no acks received"
ts=2026-02-16T14:33:33.326763978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40302"
ts=2026-02-16T14:33:33.340906491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40304"
ts=2026-02-16T14:33:34.118216191Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ee30c144 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T14:33:35.83714362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38668"
ts=2026-02-16T14:33:46.290742114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=debug ts=2026-02-16T14:33:49.123628435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:05.840293511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58780"
ts=2026-02-16T14:34:16.293672103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-81e5c3b7 10.130.0.68:7946"
level=debug ts=2026-02-16T14:34:38.508719289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:46.297215513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-81e5c3b7 10.130.0.68:7946"
ts=2026-02-16T14:35:16.300101546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:35:35.54424587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:35:35.545819821Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:35:46.303325995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-81e5c3b7 10.130.0.68:7946"
ts=2026-02-16T14:36:08.329225603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54786"
ts=2026-02-16T14:36:10.602754357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51924"
level=debug ts=2026-02-16T14:36:14.57448127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:36:16.306298262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:36:35.851221914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40876"