level=info ts=2026-02-16T05:56:26.32625322Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T05:56:26.326296918Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T05:56:26.328120642Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T05:56:26.32859766Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T05:56:26.328711559Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.60 inf=eth0
level=debug ts=2026-02-16T05:56:26.329883589Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T05:56:26.330064335Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.60 inf=eth0
level=info ts=2026-02-16T05:56:26.330212764Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-a7e144a7
ts=2026-02-16T05:56:26.330324228Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T05:56:26.330341422Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.60 advertisePort=7946
level=debug ts=2026-02-16T05:56:26.330361548Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.60 advertisePort=7946
level=info ts=2026-02-16T05:56:26.330536998Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T05:56:26.340152208Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T05:56:26.340199891Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:56:26.340215973Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.681723ms
level=error ts=2026-02-16T05:56:26.34023292Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T05:56:26.340244326Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T05:56:26.344010223Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:26.344030631Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T05:56:26.349284072Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T05:56:26.349283497Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T05:56:26.349309095Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T05:56:26.349292931Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T05:56:26.349386686Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T05:56:26.349401507Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:26.349635228Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-02-16T05:56:26.349645843Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-02-16T05:56:26.349649981Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-02-16T05:56:26.349653654Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T05:56:26.349703477Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T05:56:26.349703229Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T05:56:26.349730427Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T05:56:26.349885249Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T05:56:26.34990596Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T05:56:26.349932716Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T05:56:27.35006669Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T05:56:27.480347667Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T05:56:27.48040019Z caller=loki.go:599 msg="Loki started" startup_time=1.222837978s
ts=2026-02-16T05:56:27.574562422Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:27.574586181Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:56:29.802339049Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:29.802364851Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T05:56:32.48056519Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T05:56:32.480619537Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T05:56:36.834974723Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:36.835005582Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T05:56:37.864491159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:56:45.559067504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T05:56:45.56051275Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.58:7946: dial tcp 10.128.0.58:7946: connect: connection refused"
ts=2026-02-16T05:56:45.561094829Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
ts=2026-02-16T05:56:45.561234975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.60:7946"
ts=2026-02-16T05:56:45.561257287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57542"
level=info ts=2026-02-16T05:56:45.56157384Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=19.221324045s
ts=2026-02-16T05:56:51.676756053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51334"
level=debug ts=2026-02-16T05:56:59.049776808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:57:01.120506092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40402"
ts=2026-02-16T05:57:06.111769326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55832"
ts=2026-02-16T05:57:08.008973373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T05:57:21.356732112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:57:38.010647139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T05:57:42.274698698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:58:06.114591637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45250"
ts=2026-02-16T05:58:08.012350544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T05:58:15.449278197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T05:58:15.45116448Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T05:58:35.706757733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:44658"
ts=2026-02-16T05:58:38.014331569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T05:59:01.130838466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60318"
level=debug ts=2026-02-16T05:59:01.187070455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:59:06.11723875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38872"
ts=2026-02-16T05:59:08.016370495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T05:59:35.709459823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:39666"
ts=2026-02-16T05:59:38.018323822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:00:06.121517204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36900"
ts=2026-02-16T06:00:08.019772743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:00:09.965073178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T06:00:35.71245205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56768"
ts=2026-02-16T06:00:38.021858983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:01:01.137343884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36372"
ts=2026-02-16T06:01:08.023750943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:01:08.466842036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T06:01:38.026310091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:02:05.718542682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:33604"
ts=2026-02-16T06:02:08.027606841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:02:20.884688172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T06:02:38.029186954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:03:06.131926007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60054"
ts=2026-02-16T06:03:08.031212213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:03:12.422419573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T06:03:12.424421882Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:03:31.146673977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33862"
ts=2026-02-16T06:03:38.03337261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:04:08.034396591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:04:10.731892217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:04:35.726666465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:52304"
ts=2026-02-16T06:04:36.137425903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44594"
ts=2026-02-16T06:04:38.036158292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:04:46.825182892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:05:08.038143456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:05:21.927273165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:05:31.155534842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34462"
ts=2026-02-16T06:05:35.730977502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56128"
ts=2026-02-16T06:05:38.039445256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:06:01.156564446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51020"
level=debug ts=2026-02-16T06:06:03.959639132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:06:08.04095256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:06:31.157770829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56458"
level=info ts=2026-02-16T06:06:32.481458638Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T06:06:32.485011043Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:06:32.487155909Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:06:32.489217098Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:06:32.489249457Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:06:32.489262751Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-02-16T06:06:38.042364608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:06:48.708618099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:06:48.710294907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:07:08.044253828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:07:31.161009898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39924"
ts=2026-02-16T06:07:35.739016373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:50950"
ts=2026-02-16T06:07:38.045970541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:07:42.825664231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:08:08.047882526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:08:36.150644316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35466"
ts=2026-02-16T06:08:38.049798598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:08:38.426623246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:09:08.0507479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:09:22.288954634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:09:31.169015371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35506"
ts=2026-02-16T06:09:38.05309555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:10:08.055346062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:10:23.027378422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:10:35.750535166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57174"
ts=2026-02-16T06:10:36.158485532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35834"
ts=2026-02-16T06:10:38.05719136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:11:05.7529808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:54216"
ts=2026-02-16T06:11:08.058611788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:11:18.758065136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:11:18.760151907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:11:35.754949688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57522"
ts=2026-02-16T06:11:38.060097683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:12:03.780891978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:12:08.062474236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:12:36.166750918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36526"
ts=2026-02-16T06:12:38.063816559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:12:46.762679501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:13:05.759793742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:35800"
ts=2026-02-16T06:13:08.065816574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:13:36.169993184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60544"
level=debug ts=2026-02-16T06:13:37.034527477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:13:38.067993903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:14:01.187474778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47884"
ts=2026-02-16T06:14:08.070374656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:14:31.189232185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46748"
level=debug ts=2026-02-16T06:14:33.365583576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:14:36.174920764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51568"
ts=2026-02-16T06:14:38.072272343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:15:08.073926002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:15:18.913222461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:15:18.915236762Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:15:38.075673256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:16:05.772296789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:41670"
level=debug ts=2026-02-16T06:16:07.748948964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:16:08.077374665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=error ts=2026-02-16T06:16:32.495132127Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:16:32.497461554Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:16:32.499556085Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:16:32.49959201Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:16:38.078985151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:17:04.740345188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:17:05.776797196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56398"
ts=2026-02-16T06:17:08.080067642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:17:36.186917235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58370"
ts=2026-02-16T06:17:38.08099438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:17:57.116278684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:18:08.082416713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:18:31.203811797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57538"
ts=2026-02-16T06:18:36.191531732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43588"
ts=2026-02-16T06:18:38.083874731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:18:54.336485825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:19:01.204744196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47884"
ts=2026-02-16T06:19:05.783577687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56232"
ts=2026-02-16T06:19:06.193485161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35546"
ts=2026-02-16T06:19:08.084853323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:19:35.785884797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:50552"
ts=2026-02-16T06:19:36.195340107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47766"
level=debug ts=2026-02-16T06:19:36.467327261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:19:36.468903446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:19:38.087215777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:20:08.08879806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:20:31.274526035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:20:35.789401859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42908"
ts=2026-02-16T06:20:36.198375541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59900"
ts=2026-02-16T06:20:38.090826152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:21:05.790895038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:47894"
level=debug ts=2026-02-16T06:21:07.781200138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:21:08.092527167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:21:36.202563521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40516"
ts=2026-02-16T06:21:38.093767859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:22:01.216685676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46984"
level=debug ts=2026-02-16T06:22:07.394976021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:22:08.095818311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:22:38.097206556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:22:51.36251179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:23:01.220303921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60780"
ts=2026-02-16T06:23:08.099033867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:23:36.20860731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35980"
ts=2026-02-16T06:23:38.100585538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:23:42.082388422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:23:42.084020752Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:24:05.80192881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:34054"
ts=2026-02-16T06:24:08.102499889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:24:31.228804458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55776"
ts=2026-02-16T06:24:38.104254097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:24:39.537615564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:25:01.230160871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57766"
ts=2026-02-16T06:25:06.214941943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53808"
ts=2026-02-16T06:25:08.105836959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:25:28.593434031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:25:38.107754858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:26:01.234207811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35764"
ts=2026-02-16T06:26:08.108658307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:26:26.53459105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:26:32.491904166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:26:32.493680316Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:26:32.495528622Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:26:32.495557274Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:26:38.110151042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:27:04.800443611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:27:05.812748262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48970"
ts=2026-02-16T06:27:06.22098544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39430"
ts=2026-02-16T06:27:08.111993968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:27:38.114187399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:27:50.457157237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:27:50.458694977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:28:05.815940027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60198"
ts=2026-02-16T06:28:08.115884872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:28:38.11818844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:28:39.876415568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:29:05.819153545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48380"
ts=2026-02-16T06:29:06.226466726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55000"
ts=2026-02-16T06:29:08.120305799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:29:31.248594205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56186"
level=debug ts=2026-02-16T06:29:34.553303067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:29:38.122535829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:30:01.249950148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48554"
ts=2026-02-16T06:30:08.124564198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:30:31.251125877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41796"
ts=2026-02-16T06:30:35.823409376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:54852"
level=debug ts=2026-02-16T06:30:36.316708092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:30:38.126955012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:31:08.128257877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:31:30.282806137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:31:38.130107266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:32:01.257780306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34564"
ts=2026-02-16T06:32:08.131439064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:32:16.092350912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:32:16.094107809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:32:31.259421213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58068"
ts=2026-02-16T06:32:35.832316744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:33478"
ts=2026-02-16T06:32:38.139793588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:33:01.260322324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56402"
level=debug ts=2026-02-16T06:33:02.648465526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:33:08.142149319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:33:35.837426501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:41862"
ts=2026-02-16T06:33:38.14406856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:33:46.760826748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:34:01.263543268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43124"
ts=2026-02-16T06:34:08.145679237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:34:33.430399771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:34:36.247265624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53610"
ts=2026-02-16T06:34:38.147430367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:35:01.266469068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41690"
ts=2026-02-16T06:35:08.149615159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:35:32.302225855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:35:38.151477085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:36:05.847841634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:35300"
ts=2026-02-16T06:36:06.252288264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43594"
ts=2026-02-16T06:36:08.153732896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:36:16.770404233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:36:16.772006826Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:36:31.271472648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54682"
level=error ts=2026-02-16T06:36:32.49315335Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:36:32.494936283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:36:32.496472958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:36:32.496502493Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:36:38.155315271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:37:01.272819694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:45986"
ts=2026-02-16T06:37:08.156934006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:37:10.271491441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:37:38.158879301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:37:58.084575596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:38:01.275868591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57020"
ts=2026-02-16T06:38:06.259654809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60048"
ts=2026-02-16T06:38:08.161311262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:38:38.1632432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:38:52.601914545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:39:06.263310729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43546"
ts=2026-02-16T06:39:08.165378617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:39:30.360387917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:39:31.280455185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38930"
ts=2026-02-16T06:39:35.859601905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57962"
ts=2026-02-16T06:39:38.167618915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:40:08.170093439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:40:12.07322479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:40:12.075283351Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:40:38.172396226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:41:05.720054947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:41:08.174200178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:41:38.176380839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:41:56.695965529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:42:06.27461796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49802"
ts=2026-02-16T06:42:08.178358888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:42:36.276773529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45998"
ts=2026-02-16T06:42:38.180656092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:42:51.137929979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:43:06.2789843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50984"
ts=2026-02-16T06:43:08.183149619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:43:31.29728862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43000"
ts=2026-02-16T06:43:36.281083533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55742"
ts=2026-02-16T06:43:38.184413553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:43:46.983838226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:44:01.298104383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48506"
ts=2026-02-16T06:44:08.185957673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:44:36.285711398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56030"
ts=2026-02-16T06:44:38.188381615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:44:43.722937807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:44:43.724490368Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:45:01.302124175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51376"
ts=2026-02-16T06:45:08.190999709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:45:21.522842128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:45:38.192978901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:46:04.146683599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:46:05.879947655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:55780"
ts=2026-02-16T06:46:08.194729693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=error ts=2026-02-16T06:46:32.495299746Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:46:32.496992387Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:46:32.498802131Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T06:46:32.498834335Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:46:38.196440063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:46:42.084937156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:47:01.310111216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38902"
ts=2026-02-16T06:47:05.884428771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38028"
ts=2026-02-16T06:47:08.197644508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:47:29.470128173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:47:38.200476175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:48:06.298341482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59342"
ts=2026-02-16T06:48:08.202344652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:48:11.091819343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:48:11.093416996Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:48:36.300614195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57612"
ts=2026-02-16T06:48:38.204228427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:48:49.394094778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:49:01.317937912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51274"
ts=2026-02-16T06:49:08.205814789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:49:31.319618602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46846"
ts=2026-02-16T06:49:35.892435175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:59746"
ts=2026-02-16T06:49:38.207661192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:49:44.497427948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:50:05.894666194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56356"
ts=2026-02-16T06:50:08.20915703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:50:19.367520384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:50:38.210210096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:50:55.756855788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:51:05.898799083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:49496"
ts=2026-02-16T06:51:06.309723495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41144"
ts=2026-02-16T06:51:08.211837152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"