level=info ts=2026-04-20T14:09:37.819886943Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T14:09:37.819918908Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T14:09:37.820863306Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T14:09:37.821142242Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T14:09:37.821211018Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=debug ts=2026-04-20T14:09:37.822114342Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T14:09:37.822179289Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=info ts=2026-04-20T14:09:37.82219362Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-924b83cf
ts=2026-04-20T14:09:37.822252534Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T14:09:37.822264473Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=debug ts=2026-04-20T14:09:37.822278321Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=info ts=2026-04-20T14:09:37.822381956Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T14:09:37.827357701Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T14:09:37.827374183Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T14:09:37.827384094Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.004288ms
level=error ts=2026-04-20T14:09:37.827390948Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T14:09:37.827400647Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T14:09:37.829563145Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:09:37.829576915Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T14:09:37.835550367Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T14:09:37.835556887Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-20T14:09:37.835577866Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T14:09:37.835586077Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T14:09:37.83559991Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-20T14:09:37.835654832Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T14:09:37.835891677Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-20T14:09:37.835902586Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T14:09:37.835906307Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-20T14:09:37.835909678Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T14:09:37.835950745Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T14:09:37.835978564Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T14:09:37.835992747Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T14:09:37.836106117Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T14:09:37.836112351Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T14:09:37.836121349Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T14:09:38.837130785Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T14:09:38.939418225Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T14:09:38.939487992Z caller=loki.go:599 msg="Loki started" startup_time=1.192754814s
ts=2026-04-20T14:09:39.763716442Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:09:39.763768601Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:09:42.060637351Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:09:42.060661102Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T14:09:43.94007455Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T14:09:43.940132851Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T14:09:46.819655159Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:09:46.819682491Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T14:09:46.904757129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:09:57.278276384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55110"
ts=2026-04-20T14:10:00.933526445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50902"
ts=2026-04-20T14:10:02.408928494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-20T14:10:02.408968373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50006"
ts=2026-04-20T14:10:02.410460644Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
level=info ts=2026-04-20T14:10:02.410489951Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=24.583085024s
ts=2026-04-20T14:10:05.809640123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37620"
level=debug ts=2026-04-20T14:10:07.499776179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:10:12.243043761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44964"
ts=2026-04-20T14:10:21.996639217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:10:25.606076776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T14:10:45.502625969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:10:51.998929025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:11:09.652146097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T14:11:09.653883393Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:11:12.247748818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38984"
ts=2026-04-20T14:11:22.002254274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:11:37.546919634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54966"
ts=2026-04-20T14:11:52.005685026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:12:00.653799391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:12:03.958849998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53138"
ts=2026-04-20T14:12:12.252301642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:40768"
ts=2026-04-20T14:12:22.007655356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:12:33.962175064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54594"
ts=2026-04-20T14:12:42.253818579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47788"
ts=2026-04-20T14:12:52.008502143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:13:16.690297583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:13:22.010914806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:13:33.966953459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47862"
ts=2026-04-20T14:13:52.01305147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:14:22.015848222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:14:26.443139649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:14:42.264621424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53132"
ts=2026-04-20T14:14:52.017922033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:15:03.974564318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36792"
ts=2026-04-20T14:15:22.019611383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:15:24.996777492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:15:42.269045917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33270"
ts=2026-04-20T14:15:52.021665031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:16:20.875960134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T14:16:20.877840286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:16:22.02423021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:16:33.982886807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36230"
ts=2026-04-20T14:16:52.026827129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:17:03.986040629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56826"
ts=2026-04-20T14:17:07.57681712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34102"
level=debug ts=2026-04-20T14:17:18.690988116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:17:22.028698435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:17:52.030628042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:17:54.434291612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:18:03.992741724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39880"
ts=2026-04-20T14:18:07.583213505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39568"
ts=2026-04-20T14:18:22.032942487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:18:42.287939298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38998"
level=debug ts=2026-04-20T14:18:48.733515961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:18:52.035551352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:19:07.587258369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38420"
ts=2026-04-20T14:19:22.038682396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:19:27.742177616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:19:34.001070709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41070"
ts=2026-04-20T14:19:37.590000583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43698"
ts=2026-04-20T14:19:42.292194083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58412"
level=info ts=2026-04-20T14:19:43.940188955Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T14:19:43.943560957Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:19:43.945216717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:19:43.946774573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:19:43.946802855Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T14:19:43.946816981Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T14:19:52.042011825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:20:22.045344328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:20:26.848082166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:20:26.849729934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:20:42.2961544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58374"
ts=2026-04-20T14:20:52.047993059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:21:09.450573393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:21:22.050531099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:21:52.052830715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:22:03.200519483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:22:07.602969553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56950"
ts=2026-04-20T14:22:12.302877012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33712"
ts=2026-04-20T14:22:22.054307631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:22:44.485022549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:22:52.055336318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:23:07.607935828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44286"
ts=2026-04-20T14:23:22.057249273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:23:35.607109134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:23:42.310875036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47620"
ts=2026-04-20T14:23:52.060702309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:24:12.933133223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:24:12.935164021Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:24:22.062767935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:24:52.064748107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:24:57.851633158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:25:04.031438721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40758"
ts=2026-04-20T14:25:07.617984434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39660"
ts=2026-04-20T14:25:12.317029678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41496"
ts=2026-04-20T14:25:22.067250254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:25:34.034789378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40338"
level=debug ts=2026-04-20T14:25:48.694954265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:25:52.06934191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:26:04.037900601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52396"
ts=2026-04-20T14:26:07.622400813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41484"
ts=2026-04-20T14:26:22.071224544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:26:42.324023371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59030"
level=debug ts=2026-04-20T14:26:49.6624579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:26:52.073798117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:27:07.628116855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49318"
ts=2026-04-20T14:27:22.076737175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:27:50.243291587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:27:52.0789307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:28:07.632764698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45028"
ts=2026-04-20T14:28:22.08108897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:28:44.269601631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:28:44.271200926Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:28:52.083642597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:29:04.053306481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55120"
ts=2026-04-20T14:29:12.33627839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50966"
ts=2026-04-20T14:29:22.085350458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:29:43.086928547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:29:43.950285536Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:29:43.952210492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:29:43.954463312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:29:43.954519251Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:29:52.087995306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:30:18.053003031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:30:22.091307598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:30:42.344109885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47534"
ts=2026-04-20T14:30:52.094312334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:30:59.060124003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:31:07.646876878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34436"
ts=2026-04-20T14:31:22.096897807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:31:36.585642946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:31:52.09969638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:32:04.06970567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55820"
level=debug ts=2026-04-20T14:32:16.122413512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:32:16.124146845Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:32:22.102572668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:32:34.073272408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59854"
ts=2026-04-20T14:32:52.105339169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:33:04.076802354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58658"
ts=2026-04-20T14:33:07.657033514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52624"
ts=2026-04-20T14:33:12.358758469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:32862"
level=debug ts=2026-04-20T14:33:13.824793961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:33:22.108819719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:33:52.11069598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:34:04.082875048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:49628"
level=debug ts=2026-04-20T14:34:07.462444733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:34:12.362813466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59736"
ts=2026-04-20T14:34:22.113740448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:34:49.857725911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:34:52.116286157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:35:07.66889407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53908"
ts=2026-04-20T14:35:22.118580976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:35:29.785310739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:35:34.093742251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39232"
ts=2026-04-20T14:35:42.370391296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52166"
ts=2026-04-20T14:35:52.121802921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:36:07.768597513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:36:07.770633547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:36:12.372207246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57576"
ts=2026-04-20T14:36:22.124585256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:36:42.3737632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57880"
ts=2026-04-20T14:36:52.127216683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:37:07.027167886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:37:22.129907566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:37:42.377731935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39884"
ts=2026-04-20T14:37:52.132589946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:38:04.108277364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41624"
level=debug ts=2026-04-20T14:38:08.288972599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:38:22.134464384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:38:52.137713299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:39:09.125206739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:39:22.139866656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:39:34.116632665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37568"
level=error ts=2026-04-20T14:39:43.951381724Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:39:43.953502745Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:39:43.95544197Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:39:43.95549209Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:39:52.141947455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:40:04.119528508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43810"
level=debug ts=2026-04-20T14:40:06.490756888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:40:22.145165216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:40:52.148350218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:41:04.124673491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58890"
level=debug ts=2026-04-20T14:41:07.564093158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:41:07.565867398Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:41:12.3970514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52954"
ts=2026-04-20T14:41:22.151461641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:41:37.712638691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45522"
ts=2026-04-20T14:41:52.155105696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:42:03.758519754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:42:22.157555445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:42:52.158553957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:42:59.879659658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:43:07.720950807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43052"
ts=2026-04-20T14:43:22.159987385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:43:37.723639186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57368"
ts=2026-04-20T14:43:42.413835231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60576"
ts=2026-04-20T14:43:52.16198997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:43:54.636901768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:44:22.164900558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:44:42.417587585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45572"
ts=2026-04-20T14:44:52.166743805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:44:53.039815245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:45:04.146927941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54848"
ts=2026-04-20T14:45:22.169003818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:45:34.150137836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47906"
ts=2026-04-20T14:45:42.422648605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44544"
level=debug ts=2026-04-20T14:45:49.526107524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:45:49.528239336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:45:52.171560476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:46:07.734956441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55154"
ts=2026-04-20T14:46:12.426095795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:32946"
ts=2026-04-20T14:46:22.173515602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:46:34.156446244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48648"
level=debug ts=2026-04-20T14:46:39.943241208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:46:52.176112971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:47:04.15922633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56050"
ts=2026-04-20T14:47:07.739137183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34420"
ts=2026-04-20T14:47:22.179017497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:47:23.724005889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:47:52.180553874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:48:04.579982124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:48:22.18199482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
ts=2026-04-20T14:48:52.183456678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:49:05.429179234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:49:12.442061448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46618"
ts=2026-04-20T14:49:22.18585337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=error ts=2026-04-20T14:49:43.95153297Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:49:43.953558831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:49:43.955331244Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:49:43.95536533Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:49:52.192868967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:50:00.952733401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:50:00.954510366Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:50:22.199415676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:50:52.201391384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b783e850 10.130.0.71:7946"
level=debug ts=2026-04-20T14:50:59.903315969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:51:22.203694462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:51:52.206686172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:51:55.658313682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:52:07.76705489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44506"
ts=2026-04-20T14:52:12.459991915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56278"
ts=2026-04-20T14:52:16.3873346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35646"
ts=2026-04-20T14:52:16.399864742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35650"
ts=2026-04-20T14:52:19.823963267Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b783e850 (timeout reached)"
ts=2026-04-20T14:52:22.209496232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:52:22.824030259Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b783e850 has failed, no acks received"
ts=2026-04-20T14:52:34.194724913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42466"
ts=2026-04-20T14:52:34.823976851Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b783e850 (timeout reached)"
ts=2026-04-20T14:52:37.770233431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42570"
ts=2026-04-20T14:52:37.823670759Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b783e850 has failed, no acks received"
level=debug ts=2026-04-20T14:52:49.455874541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:52:52.211026548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T14:53:04.198580623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59582"
ts=2026-04-20T14:53:22.213567585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:53:31.411646049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:53:34.201382988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52942"
ts=2026-04-20T14:53:52.216136136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:54:22.218400272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
level=debug ts=2026-04-20T14:54:28.587643916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:54:28.589445169Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:54:34.206410867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59276"
ts=2026-04-20T14:54:52.221131741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T14:55:07.784043896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45732"
level=debug ts=2026-04-20T14:55:10.544545828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:55:22.222976743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T14:55:37.907562554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44702"
level=debug ts=2026-04-20T14:55:46.127001598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:55:52.224884851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T14:56:04.213840407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41272"
ts=2026-04-20T14:56:22.226976962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T14:56:38.773655012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:56:52.230270492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:57:07.795058046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60098"
ts=2026-04-20T14:57:22.233679307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T14:57:34.910812957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:57:52.237035013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T14:58:04.2260956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54134"
ts=2026-04-20T14:58:22.238959704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
level=debug ts=2026-04-20T14:58:34.671514091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:58:34.673138096Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:58:52.240958933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T14:59:22.242832934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
level=debug ts=2026-04-20T14:59:36.165323342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:59:37.808575458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51970"
level=error ts=2026-04-20T14:59:43.950019198Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:59:43.951964434Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:59:43.953939255Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:59:43.953974752Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:59:52.244800154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:00:04.236394555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53840"
ts=2026-04-20T15:00:07.81141247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33378"
ts=2026-04-20T15:00:22.246847358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
level=debug ts=2026-04-20T15:00:33.683150739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:00:37.813916661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47466"
ts=2026-04-20T15:00:52.247845953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T15:01:22.249305177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55bc670e 10.130.0.71:7946"
ts=2026-04-20T15:01:29.823309842Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-55bc670e (timeout reached)"
ts=2026-04-20T15:01:32.822991061Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-55bc670e has failed, no acks received"
ts=2026-04-20T15:01:34.245624911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42562"
level=debug ts=2026-04-20T15:01:35.959293662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:01:37.819342196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53674"
ts=2026-04-20T15:01:44.82403489Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-55bc670e (timeout reached)"
ts=2026-04-20T15:01:47.823455001Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-55bc670e has failed, no acks received"
ts=2026-04-20T15:01:49.084841794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48352"
ts=2026-04-20T15:01:49.099172763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48354"
ts=2026-04-20T15:01:52.251227794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:01:52.823115933Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-55bc670e as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-20T15:01:54.823732994Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-55bc670e (timeout reached)"
ts=2026-04-20T15:01:57.823510949Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-55bc670e has failed, no acks received"
ts=2026-04-20T15:02:04.248975716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35846"
ts=2026-04-20T15:02:22.253205431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
ts=2026-04-20T15:02:25.267410867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52950"
level=debug ts=2026-04-20T15:02:37.365720738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:02:52.25596464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:02:55.269142654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57058"
ts=2026-04-20T15:03:04.256057981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44846"
ts=2026-04-20T15:03:07.829104594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50332"
ts=2026-04-20T15:03:22.258293067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
level=debug ts=2026-04-20T15:03:23.778595651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:03:23.780415937Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:03:25.270809755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59316"
ts=2026-04-20T15:03:37.83181061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35952"
ts=2026-04-20T15:03:52.260182827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
ts=2026-04-20T15:03:55.272626628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53984"
level=debug ts=2026-04-20T15:04:01.075276862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:04:22.262712271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:04:37.836058206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59160"
level=debug ts=2026-04-20T15:04:44.672561631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:04:52.266222235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:05:04.270937344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58420"
ts=2026-04-20T15:05:22.271567327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
level=debug ts=2026-04-20T15:05:32.597717854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:05:34.275091528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40244"
ts=2026-04-20T15:05:52.272752169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
ts=2026-04-20T15:06:07.842798574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43060"
level=debug ts=2026-04-20T15:06:09.585279127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:06:22.275045173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:06:52.278768335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T15:07:03.844162665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:07:03.84615823Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:07:22.280984198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
ts=2026-04-20T15:07:52.28232721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
level=debug ts=2026-04-20T15:08:05.890887422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:08:07.853832649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40558"
ts=2026-04-20T15:08:22.284129945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-eaae84da 10.130.0.71:7946"
level=debug ts=2026-04-20T15:08:46.886102754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:08:52.286208082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:09:07.859002491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55818"
ts=2026-04-20T15:09:22.289030028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:09:25.304637506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52580"
level=debug ts=2026-04-20T15:09:28.563125694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:09:34.824224314Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-eaae84da (timeout reached)"
ts=2026-04-20T15:09:37.82342947Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-eaae84da has failed, no acks received"
ts=2026-04-20T15:09:37.861610889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39772"
level=error ts=2026-04-20T15:09:43.953788854Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:09:43.95615265Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:09:43.95842583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:09:43.958465689Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:09:44.823105032Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-eaae84da (timeout reached)"
ts=2026-04-20T15:09:47.823665356Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-eaae84da has failed, no acks received"
ts=2026-04-20T15:09:52.292146272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:09:54.824113661Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-eaae84da (timeout reached)"
ts=2026-04-20T15:09:57.823768443Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-eaae84da has failed, no acks received"
ts=2026-04-20T15:10:02.122419579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55354"
ts=2026-04-20T15:10:02.135975767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55368"
ts=2026-04-20T15:10:04.298726827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44260"
level=debug ts=2026-04-20T15:10:20.447905541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:10:22.294609694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:10:52.297010203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:11:13.199302359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57392"
level=debug ts=2026-04-20T15:11:14.98659905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:11:14.988118254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:11:22.299818889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:11:52.302571655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:12:07.87520359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45364"
level=debug ts=2026-04-20T15:12:08.593199459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:12:22.305833828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:12:34.310412083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36146"
ts=2026-04-20T15:12:37.877658667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50152"
level=debug ts=2026-04-20T15:12:43.80141813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:12:52.30903531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:13:22.310694873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
level=debug ts=2026-04-20T15:13:22.808270558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:13:43.214650226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47406"
ts=2026-04-20T15:13:52.312887783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T15:14:11.097290018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:14:22.315106205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:14:43.219743589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:36982"
ts=2026-04-20T15:14:52.317340926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:15:07.8909806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45646"
level=debug ts=2026-04-20T15:15:11.407172883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:15:11.409212617Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:15:22.319371604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:15:34.328245146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33290"
ts=2026-04-20T15:15:37.893974682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54934"
ts=2026-04-20T15:15:43.22453302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58582"
ts=2026-04-20T15:15:52.320290729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:16:07.89732528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48746"
level=debug ts=2026-04-20T15:16:09.330889502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:16:13.225702767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44994"
ts=2026-04-20T15:16:22.322258211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:16:37.900647187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40306"
ts=2026-04-20T15:16:52.32383806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:17:04.337415387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33136"
level=debug ts=2026-04-20T15:17:07.100169172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:17:07.904010837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47344"
ts=2026-04-20T15:17:22.325037785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:17:34.340594996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:32824"
ts=2026-04-20T15:17:43.233645804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38410"
ts=2026-04-20T15:17:52.326387513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
level=debug ts=2026-04-20T15:18:03.200614604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:18:22.327779167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:18:34.345623295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57724"
ts=2026-04-20T15:18:37.912521366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43206"
level=debug ts=2026-04-20T15:18:49.650398879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:18:52.329278567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:19:04.348650258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44094"
ts=2026-04-20T15:19:07.915304311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33684"
ts=2026-04-20T15:19:22.331280398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:19:34.351986116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41078"
level=debug ts=2026-04-20T15:19:39.109843545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:19:39.111706948Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:19:43.95018309Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:19:43.951942182Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:19:43.9542379Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:19:43.954271262Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:19:52.33416944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:20:22.337453124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T15:20:41.000789936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:20:43.25085868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:51952"
ts=2026-04-20T15:20:52.340093533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:21:22.342031852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T15:21:23.770687336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:21:34.362488092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55436"
ts=2026-04-20T15:21:43.25557097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:37402"
ts=2026-04-20T15:21:52.344794252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:22:04.365981387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56060"
level=debug ts=2026-04-20T15:22:20.033873634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:22:22.347042182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b871830b 10.130.0.71:7946"
ts=2026-04-20T15:22:37.932008912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55990"
ts=2026-04-20T15:22:52.349339186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T15:23:00.122570457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:23:07.934523959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48796"
ts=2026-04-20T15:23:22.351966498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:23:34.374878762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45500"
level=debug ts=2026-04-20T15:23:45.823440281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:23:45.825216798Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:23:52.354563752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:24:09.784179212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41158"
ts=2026-04-20T15:24:09.795931131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41168"
ts=2026-04-20T15:24:22.356435983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:24:24.824077309Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b871830b (timeout reached)"
ts=2026-04-20T15:24:27.823998372Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b871830b has failed, no acks received"
level=debug ts=2026-04-20T15:24:30.715720129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:24:34.38000961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44754"
ts=2026-04-20T15:24:39.779753295Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b871830b as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-20T15:24:52.358766045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:25:07.943268672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49212"
level=debug ts=2026-04-20T15:25:20.455817708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:25:22.36099938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:25:24.276912384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:54168"
ts=2026-04-20T15:25:52.363176406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:25:54.278844458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48482"
ts=2026-04-20T15:26:04.390181874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37798"
level=debug ts=2026-04-20T15:26:04.858806289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:26:22.365647486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:26:34.393183421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47400"
ts=2026-04-20T15:26:52.368527985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
level=debug ts=2026-04-20T15:26:55.480397581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:27:22.371579172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:27:24.287040928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60114"
ts=2026-04-20T15:27:37.956356711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54552"
level=debug ts=2026-04-20T15:27:39.018179669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:27:39.02024329Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:27:52.374319439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:28:22.376293371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:28:24.290779566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34772"
level=debug ts=2026-04-20T15:28:26.02614022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:28:34.405321945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35850"
ts=2026-04-20T15:28:52.378729869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T15:29:02.80943837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:29:07.96371887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52124"
ts=2026-04-20T15:29:22.384333563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:29:24.295010874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58066"
level=debug ts=2026-04-20T15:29:42.80736572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:29:43.94961736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:29:43.951197943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:29:43.952680146Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T15:29:43.952705975Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:29:52.386859065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:29:54.296315034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47346"
ts=2026-04-20T15:30:22.389706055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:30:37.97311796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36676"
level=debug ts=2026-04-20T15:30:38.190034177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:30:52.391379399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:31:22.393763515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:31:24.304209305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45508"
level=debug ts=2026-04-20T15:31:33.797855538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:31:33.799455929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:31:37.978040801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38290"
ts=2026-04-20T15:31:52.396255027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
ts=2026-04-20T15:32:04.421985523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40698"
level=debug ts=2026-04-20T15:32:12.5319833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:32:22.399270755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:32:34.425280245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40314"
ts=2026-04-20T15:32:37.983157838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56098"
ts=2026-04-20T15:32:52.40122159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
level=debug ts=2026-04-20T15:33:01.66343793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:33:07.98635513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42058"
ts=2026-04-20T15:33:22.404054468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:33:24.314620391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42858"
level=debug ts=2026-04-20T15:33:37.392247836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:33:37.98953752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54374"
ts=2026-04-20T15:33:52.407394829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9qpqm-5631ee3e 10.129.0.75:7946"
level=debug ts=2026-04-20T15:34:15.326195137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:34:22.409797279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
ts=2026-04-20T15:34:24.31811091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58454"
ts=2026-04-20T15:34:52.411012415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-17fcea77 10.130.0.71:7946"
level=debug ts=2026-04-20T15:34:54.552062071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T15:34:54.553848846Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T15:35:04.439875213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42778"
ts=2026-04-20T15:35:22.413509709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:35:34.442446003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34602"
ts=2026-04-20T15:35:37.999414713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54700"
level=debug ts=2026-04-20T15:35:39.206782949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T15:35:52.416216914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-rpnp9-09e3d271 10.128.0.90:7946"
ts=2026-04-20T15:36:08.002264101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33308"