level=info ts=2026-04-23T13:19:13.897993091Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-23T13:19:13.898019389Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-23T13:19:13.898898163Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-23T13:19:13.899243039Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T13:19:13.899335655Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=debug ts=2026-04-23T13:19:13.90056631Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T13:19:13.900643734Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=info ts=2026-04-23T13:19:13.900851074Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f1ceb1bd
ts=2026-04-23T13:19:13.900942097Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-23T13:19:13.900962515Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=debug ts=2026-04-23T13:19:13.900988286Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=info ts=2026-04-23T13:19:13.901141555Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-23T13:19:13.910099446Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T13:19:13.910120895Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-23T13:19:13.91013428Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.995813ms
level=error ts=2026-04-23T13:19:13.910144186Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-23T13:19:13.910159062Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-23T13:19:13.914274492Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T13:19:13.914295729Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T13:19:13.916398774Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-23T13:19:13.916407939Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-23T13:19:13.916433971Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-23T13:19:13.916471297Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-23T13:19:13.91648516Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-23T13:19:13.916486819Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-23T13:19:13.916499443Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-23T13:19:13.916741991Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-23T13:19:13.916750065Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-23T13:19:13.916834894Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-23T13:19:13.916909594Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-23T13:19:13.916909292Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-23T13:19:13.916936923Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-23T13:19:13.917115585Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-23T13:19:13.917129961Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-23T13:19:13.917159163Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-23T13:19:14.918272268Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-23T13:19:15.038529298Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-23T13:19:15.038582471Z caller=loki.go:599 msg="Loki started" startup_time=1.213268422s
ts=2026-04-23T13:19:15.653934318Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T13:19:15.653967947Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:19:19.033137577Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T13:19:19.033158783Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T13:19:20.038835659Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-23T13:19:20.038898622Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-23T13:19:24.527519462Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T13:19:24.527542546Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-23T13:19:30.392401042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:19:36.178393136Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T13:19:36.178417705Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:19:42.416581149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:52336"
ts=2026-04-23T13:19:47.431502378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36978"
level=debug ts=2026-04-23T13:19:50.859926265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:19:52.997164024Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.76:7946: dial tcp 10.129.0.76:7946: connect: connection refused"
ts=2026-04-23T13:19:52.997971569Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.73:7946: dial tcp 10.129.0.73:7946: connect: connection refused"
ts=2026-04-23T13:19:52.998078216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
ts=2026-04-23T13:19:52.998193698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46094"
ts=2026-04-23T13:19:52.998558166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.72:7946"
ts=2026-04-23T13:19:52.999815534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-23T13:19:53.001479435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.83:7946"
ts=2026-04-23T13:19:53.004625545Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.84:7946: dial tcp 10.130.0.84:7946: connect: connection refused"
level=info ts=2026-04-23T13:19:53.004637364Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=39.094471733s
ts=2026-04-23T13:19:57.579247678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37896"
ts=2026-04-23T13:19:58.332688069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:20:09.576247812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53662"
level=debug ts=2026-04-23T13:20:10.75158762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:20:28.336058093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:20:36.804786034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:20:58.338026808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:21:12.44875689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T13:21:12.450433906Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:21:23.208962583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57460"
ts=2026-04-23T13:21:28.33997746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:21:52.732551353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:21:53.209909788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37100"
ts=2026-04-23T13:21:58.342033211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:22:17.441663209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60640"
ts=2026-04-23T13:22:23.211199877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40826"
ts=2026-04-23T13:22:28.343294429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:22:53.020244286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:22:58.344289843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:23:17.446534588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53490"
ts=2026-04-23T13:23:23.216156769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37914"
ts=2026-04-23T13:23:28.347009867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:23:35.360627515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39658"
level=debug ts=2026-04-23T13:23:44.508792532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:23:58.349404647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:24:28.350426753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:24:44.519332427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T13:24:58.352392302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:25:28.355043571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:25:49.718732957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T13:25:49.720633048Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:25:53.22457366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55218"
ts=2026-04-23T13:25:58.35812379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:26:05.376408832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51916"
ts=2026-04-23T13:26:28.359990338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:26:35.151628799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:26:47.465104067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42742"
ts=2026-04-23T13:26:53.228980479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48688"
ts=2026-04-23T13:26:58.362213196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:27:28.364287191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:27:28.744875954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:27:35.385120955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33154"
ts=2026-04-23T13:27:58.365715878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:28:17.46975536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37846"
level=debug ts=2026-04-23T13:28:26.755053874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:28:28.366742781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:28:53.236753538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56888"
ts=2026-04-23T13:28:58.368012168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:29:08.071445915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-23T13:29:20.03933234Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-23T13:29:20.042950537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:29:20.044923849Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:29:20.046880217Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:29:20.046905405Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T13:29:20.046916933Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-23T13:29:28.368746633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:29:58.37114275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:30:01.873496667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:30:01.875306749Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:30:17.47620769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44620"
ts=2026-04-23T13:30:23.244127647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58274"
ts=2026-04-23T13:30:28.373955967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:30:46.386406367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:30:47.477375134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60392"
ts=2026-04-23T13:30:53.245072975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34768"
ts=2026-04-23T13:30:58.377362352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:31:28.380224268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:31:44.583970814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:31:53.250443504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35112"
ts=2026-04-23T13:31:58.382142201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:32:23.251620932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49066"
ts=2026-04-23T13:32:28.384350166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:32:31.739769802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:32:35.417963077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53134"
ts=2026-04-23T13:32:47.482791099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46294"
ts=2026-04-23T13:32:53.25326824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40014"
ts=2026-04-23T13:32:58.387074168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:33:05.420705904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54670"
level=debug ts=2026-04-23T13:33:21.820904669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:33:23.254441454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41726"
ts=2026-04-23T13:33:28.388993559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:33:35.423375181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48774"
ts=2026-04-23T13:33:58.390909051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:34:01.599746336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:34:01.601674908Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:34:05.426305348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57048"
ts=2026-04-23T13:34:28.393504015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:34:50.006119349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:34:53.261772202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38026"
ts=2026-04-23T13:34:58.395559308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:35:28.397140204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:35:45.348295492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:35:47.494788771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55300"
ts=2026-04-23T13:35:58.398680332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:36:28.401005366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:36:32.330946186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:36:58.402910438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:37:05.443776351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60646"
ts=2026-04-23T13:37:17.502104759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45544"
ts=2026-04-23T13:37:28.404465793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:37:28.646689971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:37:58.405468979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:38:23.281489829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52194"
ts=2026-04-23T13:38:28.408213556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:38:29.936352705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:38:29.938241469Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:38:35.45291071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57640"
ts=2026-04-23T13:38:58.410070277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
level=debug ts=2026-04-23T13:39:18.023633724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:39:20.049929031Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:39:20.051889401Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:39:20.053778189Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:39:20.053824974Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:39:28.412155488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:39:53.290658183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59800"
ts=2026-04-23T13:39:58.415668757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:40:03.415168249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:40:17.515829449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44278"
ts=2026-04-23T13:40:28.41844173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:40:53.293142604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52140"
ts=2026-04-23T13:40:58.419778757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:41:03.20236998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:41:28.421099821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:41:47.329693643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:41:47.523030177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47422"
ts=2026-04-23T13:41:53.296267817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53028"
ts=2026-04-23T13:41:58.422192576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:42:05.47675277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58944"
level=debug ts=2026-04-23T13:42:27.189312284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:42:27.191514391Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:42:28.423342513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:42:53.298439581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56376"
ts=2026-04-23T13:42:58.425299973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:43:04.732044926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:43:28.427851724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:43:47.533368328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39832"
level=debug ts=2026-04-23T13:43:55.945825781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:43:58.428710676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:44:05.48892207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56318"
ts=2026-04-23T13:44:28.431378525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:44:35.492549243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34094"
level=debug ts=2026-04-23T13:44:42.780548701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:44:58.433868535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:45:17.538490648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45932"
ts=2026-04-23T13:45:28.435652153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:45:30.072874264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:45:35.502865113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51572"
ts=2026-04-23T13:45:47.540293636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38482"
ts=2026-04-23T13:45:53.308657389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53660"
ts=2026-04-23T13:45:58.437474654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:46:05.505887674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55008"
ts=2026-04-23T13:46:17.541962736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53232"
ts=2026-04-23T13:46:28.439871322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:46:28.64398166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:46:28.645571832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:46:35.509145262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41706"
ts=2026-04-23T13:46:58.441683484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:47:17.544931493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58534"
level=debug ts=2026-04-23T13:47:17.743512296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:47:28.442892652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e604bdc5 10.129.0.78:7946"
ts=2026-04-23T13:47:35.516907851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54768"
ts=2026-04-23T13:47:53.318500065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37738"
ts=2026-04-23T13:47:58.444902894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:48:02.056698386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:48:05.520113827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34822"
ts=2026-04-23T13:48:28.446567421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:48:37.019175352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54870"
ts=2026-04-23T13:48:37.028538718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54872"
level=debug ts=2026-04-23T13:48:42.439956755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:48:45.902982435Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e604bdc5 (timeout reached)"
ts=2026-04-23T13:48:47.549970458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42050"
ts=2026-04-23T13:48:48.902500828Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e604bdc5 has failed, no acks received"
ts=2026-04-23T13:48:58.447961559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:49:00.902659401Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e604bdc5 (timeout reached)"
ts=2026-04-23T13:49:03.90259902Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e604bdc5 has failed, no acks received"
ts=2026-04-23T13:49:08.902612459Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e604bdc5 as failed, suspect timeout reached (2 peer confirmations)"
level=error ts=2026-04-23T13:49:20.05100367Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:49:20.05350422Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:49:20.055999372Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:49:20.056034983Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:49:28.449870168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:49:31.133405745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56590"
level=debug ts=2026-04-23T13:49:41.385289089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:49:47.554588378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55198"
ts=2026-04-23T13:49:58.452145813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
ts=2026-04-23T13:50:28.45346772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
level=debug ts=2026-04-23T13:50:32.42270943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:50:32.424523477Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:50:35.536886965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38580"
ts=2026-04-23T13:50:47.557140627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38480"
ts=2026-04-23T13:50:58.455127166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:51:17.558416193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48612"
ts=2026-04-23T13:51:28.456603894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
level=debug ts=2026-04-23T13:51:29.731246797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:51:35.543212686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38050"
ts=2026-04-23T13:51:58.457541508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T13:52:25.13198461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:52:28.458699374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
ts=2026-04-23T13:52:35.549820614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34788"
ts=2026-04-23T13:52:47.562512733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60292"
ts=2026-04-23T13:52:58.461439128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:53:01.149075353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42806"
ts=2026-04-23T13:53:05.552678439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43688"
level=debug ts=2026-04-23T13:53:13.916165305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:53:17.5641807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38430"
ts=2026-04-23T13:53:28.463597499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
ts=2026-04-23T13:53:58.465076976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
level=debug ts=2026-04-23T13:54:14.404253093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:54:17.568204409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60534"
ts=2026-04-23T13:54:28.466528458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
level=debug ts=2026-04-23T13:54:49.694329965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:54:49.696730326Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:54:58.469097873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:55:28.471852051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1028a83d 10.129.0.78:7946"
ts=2026-04-23T13:55:31.159349485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44852"
ts=2026-04-23T13:55:35.568581282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58152"
ts=2026-04-23T13:55:35.902983497Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1028a83d (timeout reached)"
ts=2026-04-23T13:55:38.902921877Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1028a83d has failed, no acks received"
level=debug ts=2026-04-23T13:55:50.97508041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:55:55.902993572Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1028a83d (timeout reached)"
ts=2026-04-23T13:55:58.473459026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:55:58.902488263Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1028a83d has failed, no acks received"
ts=2026-04-23T13:55:58.903549442Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1028a83d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T13:56:18.300367761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40558"
ts=2026-04-23T13:56:18.310747582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40560"
ts=2026-04-23T13:56:28.474470533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
level=debug ts=2026-04-23T13:56:30.937241034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:56:58.475894412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
ts=2026-04-23T13:57:01.681289729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35734"
level=debug ts=2026-04-23T13:57:15.5850793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:57:17.586983009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34918"
ts=2026-04-23T13:57:28.47741291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:57:35.581577905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47230"
ts=2026-04-23T13:57:58.479603661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T13:58:13.976872807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:58:28.482595891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T13:58:35.587644206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49266"
ts=2026-04-23T13:58:47.597257646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53402"
level=debug ts=2026-04-23T13:58:53.252495933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:58:53.254410773Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:58:58.484704318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T13:59:05.590813319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:46092"
ts=2026-04-23T13:59:17.598367223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45960"
level=error ts=2026-04-23T13:59:20.051297647Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:59:20.053239836Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:59:20.055107478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:59:20.055135356Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:59:28.485790664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
level=debug ts=2026-04-23T13:59:29.653485726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:59:31.692042358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59552"
ts=2026-04-23T13:59:58.487639182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:00:01.69343837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42888"
ts=2026-04-23T14:00:05.59691044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51482"
level=debug ts=2026-04-23T14:00:17.500791937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:00:28.490939573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:00:58.492693256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
ts=2026-04-23T14:01:01.696185863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40708"
level=debug ts=2026-04-23T14:01:07.81951994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:01:28.493730457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:01:47.610895299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49678"
ts=2026-04-23T14:01:58.495450251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
level=debug ts=2026-04-23T14:02:05.605246427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:02:28.497672762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:02:42.502140258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:02:42.504335781Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:02:58.500101047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:03:05.615474285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37280"
level=debug ts=2026-04-23T14:03:21.287819374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:03:28.502145058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:03:47.618054025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42986"
ts=2026-04-23T14:03:58.504219217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:04:13.522254854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:04:28.505611242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dc484d47 10.129.0.78:7946"
ts=2026-04-23T14:04:35.626441094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37118"
ts=2026-04-23T14:04:47.620949201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59694"
ts=2026-04-23T14:04:55.902329629Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-dc484d47 (timeout reached)"
level=debug ts=2026-04-23T14:04:57.700590117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:04:58.507591716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:04:58.90267225Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-dc484d47 has failed, no acks received"
ts=2026-04-23T14:05:02.618153275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34588"
ts=2026-04-23T14:05:02.6277214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34602"
ts=2026-04-23T14:05:05.902622999Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-dc484d47 (timeout reached)"
ts=2026-04-23T14:05:08.902719297Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-dc484d47 has failed, no acks received"
ts=2026-04-23T14:05:10.903021192Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-dc484d47 (timeout reached)"
ts=2026-04-23T14:05:13.903222642Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-dc484d47 has failed, no acks received"
ts=2026-04-23T14:05:14.385450041Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-dc484d47 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T14:05:28.509944735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:05:35.632718952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38366"
ts=2026-04-23T14:05:50.849876207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44704"
level=debug ts=2026-04-23T14:05:52.908686607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:05:58.512663434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:06:20.85113303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54904"
ts=2026-04-23T14:06:28.514904822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-59c45aa0 10.129.0.78:7946"
ts=2026-04-23T14:06:47.63129093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37526"
ts=2026-04-23T14:06:50.852453067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45248"
level=debug ts=2026-04-23T14:06:50.90765794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:06:50.909438879Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:06:58.517766304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:07:20.853856643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42920"
ts=2026-04-23T14:07:28.519698396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-59c45aa0 10.129.0.78:7946"
ts=2026-04-23T14:07:47.636631326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46910"
level=debug ts=2026-04-23T14:07:48.59500676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:07:50.855464666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46466"
ts=2026-04-23T14:07:58.521213905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:08:17.637698011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56204"
ts=2026-04-23T14:08:28.522224151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:08:29.839405814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:08:35.651565475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:46514"
ts=2026-04-23T14:08:58.524211824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:09:17.641792519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50610"
level=error ts=2026-04-23T14:09:20.049897732Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:09:20.051698324Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:09:20.053503669Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:09:20.05353395Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T14:09:20.686971968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:09:20.861380093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36586"
ts=2026-04-23T14:09:28.52653661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:09:58.528517731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-59c45aa0 10.129.0.78:7946"
level=debug ts=2026-04-23T14:10:21.131999393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:10:28.529841039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:10:35.663748076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54056"
ts=2026-04-23T14:10:47.649463559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40752"
ts=2026-04-23T14:10:58.531117797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:11:19.184860423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:11:19.186880241Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:11:20.866696382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59948"
ts=2026-04-23T14:11:28.533790676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:11:35.902335265Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-59c45aa0 (timeout reached)"
ts=2026-04-23T14:11:38.903148115Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-59c45aa0 has failed, no acks received"
ts=2026-04-23T14:11:45.902864896Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-59c45aa0 (timeout reached)"
ts=2026-04-23T14:11:48.902733967Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-59c45aa0 has failed, no acks received"
ts=2026-04-23T14:11:58.537484687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:12:08.711197196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33126"
ts=2026-04-23T14:12:08.720858859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33138"
level=debug ts=2026-04-23T14:12:12.106741778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:12:17.657404197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43572"
ts=2026-04-23T14:12:28.540001065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:12:35.676327018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39504"
ts=2026-04-23T14:12:47.658998571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33806"
ts=2026-04-23T14:12:58.541981704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:13:01.978559576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:13:28.544664302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
level=debug ts=2026-04-23T14:13:38.347403957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:13:44.179467101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49888"
ts=2026-04-23T14:13:47.66188044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35506"
ts=2026-04-23T14:13:58.546542318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
ts=2026-04-23T14:14:05.685057781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57354"
ts=2026-04-23T14:14:14.180640175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41860"
ts=2026-04-23T14:14:28.547599188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:14:34.713321853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:14:58.549489252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
level=debug ts=2026-04-23T14:15:14.135202271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:15:14.136821335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:15:28.552032902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:15:58.554820405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:16:00.043886493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:16:28.556980401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:16:44.188768586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58954"
level=debug ts=2026-04-23T14:16:48.53615228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:16:58.557915615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
ts=2026-04-23T14:17:28.559201701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
level=debug ts=2026-04-23T14:17:31.815456737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:17:58.56071852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:18:14.193906282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38342"
level=debug ts=2026-04-23T14:18:18.000142466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:18:28.562894779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:18:35.714902166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44460"
ts=2026-04-23T14:18:58.565885836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:19:05.717959735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38100"
level=debug ts=2026-04-23T14:19:05.82652332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:19:05.828361858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:19:14.199007525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36480"
level=error ts=2026-04-23T14:19:20.050919644Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:19:20.053025288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:19:20.05478069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:19:20.054825727Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:19:28.568990295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:19:35.721641004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57060"
level=debug ts=2026-04-23T14:19:46.164346756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:19:58.57104163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:20:28.57281601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
level=debug ts=2026-04-23T14:20:41.18716468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:20:44.206378415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51370"
ts=2026-04-23T14:20:58.574092609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-428cbab3 10.129.0.78:7946"
ts=2026-04-23T14:21:17.706355373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50304"
level=debug ts=2026-04-23T14:21:20.184191863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:21:28.5762416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:21:30.902669147Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-428cbab3 (timeout reached)"
ts=2026-04-23T14:21:33.902064969Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-428cbab3 has failed, no acks received"
ts=2026-04-23T14:21:35.736607819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42206"
ts=2026-04-23T14:21:45.9021167Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-428cbab3 (timeout reached)"
ts=2026-04-23T14:21:47.707830216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54418"
ts=2026-04-23T14:21:48.901536377Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-428cbab3 has failed, no acks received"
ts=2026-04-23T14:21:53.902225151Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-428cbab3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T14:21:58.579343349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:22:07.990866759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:22:22.606724496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38094"
ts=2026-04-23T14:22:22.616029412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38106"
ts=2026-04-23T14:22:28.581819642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
ts=2026-04-23T14:22:35.743489044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51164"
level=debug ts=2026-04-23T14:22:50.901413899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:22:50.903302208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:22:58.583353486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:23:28.58616103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:23:41.578952602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:23:51.40802549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55438"
ts=2026-04-23T14:23:58.5897282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:24:25.005046077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:24:28.5920998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
ts=2026-04-23T14:24:51.411054041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59428"
ts=2026-04-23T14:24:58.593715539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
level=debug ts=2026-04-23T14:25:18.133612043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:25:21.41236454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49998"
ts=2026-04-23T14:25:28.594984858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
ts=2026-04-23T14:25:35.762356616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40940"
ts=2026-04-23T14:25:58.59673604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
level=debug ts=2026-04-23T14:26:05.732259804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:26:05.76499406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56470"
ts=2026-04-23T14:26:17.732821631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54706"
ts=2026-04-23T14:26:21.417319705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35866"
ts=2026-04-23T14:26:28.599490144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:26:35.768657419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54940"
level=debug ts=2026-04-23T14:26:56.222639382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:26:56.224144129Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:26:58.602893982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:27:17.73532558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41794"
ts=2026-04-23T14:27:28.604780741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
level=debug ts=2026-04-23T14:27:30.613318468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:27:35.775562896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39080"
ts=2026-04-23T14:27:47.737107663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33304"
ts=2026-04-23T14:27:58.606174673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5d05f9ea 10.129.0.78:7946"
level=debug ts=2026-04-23T14:28:20.33790927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:28:28.607626855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:28:47.739772434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49348"
ts=2026-04-23T14:28:58.609531034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:29:01.946543988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:29:20.051431324Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:29:20.053500725Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:29:20.055481652Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:29:20.055518838Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:29:21.433356129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50538"
ts=2026-04-23T14:29:28.612787454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:29:35.78829423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43722"
ts=2026-04-23T14:29:47.744389766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39578"
level=debug ts=2026-04-23T14:29:57.525492395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:29:58.614887929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:30:05.791928579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44818"
ts=2026-04-23T14:30:10.902004001Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5d05f9ea (timeout reached)"
ts=2026-04-23T14:30:13.901405745Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5d05f9ea has failed, no acks received"
ts=2026-04-23T14:30:23.615996186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56914"
ts=2026-04-23T14:30:23.62633241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56918"
ts=2026-04-23T14:30:25.902544416Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5d05f9ea (timeout reached)"
ts=2026-04-23T14:30:28.616316752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:30:28.902827448Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5d05f9ea has failed, no acks received"
ts=2026-04-23T14:30:33.901591116Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-5d05f9ea as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T14:30:35.794838848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53256"
level=debug ts=2026-04-23T14:30:37.76771594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:30:37.769288471Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:30:58.617895597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-056e2dfc 10.129.0.78:7946"
ts=2026-04-23T14:31:17.749768138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51776"
level=debug ts=2026-04-23T14:31:20.4467504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:31:27.031491798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50316"
ts=2026-04-23T14:31:28.620335573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:31:47.750617627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33374"
ts=2026-04-23T14:31:57.032530276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49678"
ts=2026-04-23T14:31:58.623323355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:32:11.840382609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:32:17.752187574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46092"
ts=2026-04-23T14:32:28.62632379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:32:58.628051347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:33:09.632464237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:33:27.036757095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49362"
ts=2026-04-23T14:33:28.629673124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-056e2dfc 10.129.0.78:7946"
ts=2026-04-23T14:33:57.038366815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33178"
ts=2026-04-23T14:33:58.631376262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:34:02.187250253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:34:05.816192668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43074"
ts=2026-04-23T14:34:28.633950064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:34:35.819783422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60598"
level=debug ts=2026-04-23T14:34:39.774135551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:34:39.775911456Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:34:47.762312696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58442"
ts=2026-04-23T14:34:57.041193882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47950"
ts=2026-04-23T14:34:58.637531791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:35:05.824580664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50570"
ts=2026-04-23T14:35:28.640543844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:35:31.605424249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:35:47.76631158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50364"
ts=2026-04-23T14:35:58.642581949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-056e2dfc 10.129.0.78:7946"
ts=2026-04-23T14:36:17.767559083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:32818"
level=debug ts=2026-04-23T14:36:25.818281967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:36:28.643760319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:36:58.645007095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:37:05.773437183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:37:28.647074083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:37:35.839277649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44942"
ts=2026-04-23T14:37:58.649599195Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-056e2dfc failed: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-04-23T14:38:00.901578512Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-056e2dfc (timeout reached)"
level=debug ts=2026-04-23T14:38:00.906168502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:38:03.90199904Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-056e2dfc has failed, no acks received"
ts=2026-04-23T14:38:05.842314905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51300"
ts=2026-04-23T14:38:10.901993357Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-056e2dfc (timeout reached)"
ts=2026-04-23T14:38:11.605479019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41758"
ts=2026-04-23T14:38:11.615872433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41762"
ts=2026-04-23T14:38:13.901658428Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-056e2dfc has failed, no acks received"
ts=2026-04-23T14:38:17.775660313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54384"
ts=2026-04-23T14:38:18.925339018Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-056e2dfc as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T14:38:28.650964793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:38:53.228419363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:38:53.230172756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:38:55.00957712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37058"
ts=2026-04-23T14:38:58.653855145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=error ts=2026-04-23T14:39:20.050616131Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:39:20.052377464Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:39:20.054015562Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:39:20.054039715Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:39:25.011101856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47190"
ts=2026-04-23T14:39:28.65711279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:39:36.429889518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:39:47.779581574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54398"
ts=2026-04-23T14:39:58.659035155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-72706aa0 10.129.0.78:7946"
ts=2026-04-23T14:40:28.661447202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:40:33.090579872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:40:58.664128615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:41:15.530927893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:41:17.783617233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51520"
ts=2026-04-23T14:41:25.020585415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59044"
ts=2026-04-23T14:41:28.665424801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:41:49.868764538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:41:58.666460335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:42:28.667325184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:42:30.579113642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:42:30.581061936Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:42:35.871740645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58626"
ts=2026-04-23T14:42:55.024775963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60724"
ts=2026-04-23T14:42:58.669980035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:43:05.874563912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58710"
ts=2026-04-23T14:43:17.792275703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59962"
level=debug ts=2026-04-23T14:43:24.718006246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:43:28.671916249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:43:35.87775278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50794"
ts=2026-04-23T14:43:55.026962871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52304"
ts=2026-04-23T14:43:58.673311555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
level=debug ts=2026-04-23T14:44:23.879072009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:44:25.028489317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34852"
ts=2026-04-23T14:44:28.676129724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:44:35.883721405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34568"
ts=2026-04-23T14:44:58.678341107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-72706aa0 10.129.0.78:7946"
level=debug ts=2026-04-23T14:45:05.304893167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:45:05.886518063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49134"
ts=2026-04-23T14:45:28.680951706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:45:55.03412766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33706"
level=debug ts=2026-04-23T14:45:55.873705904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:45:58.682953071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-72706aa0 10.129.0.78:7946"
ts=2026-04-23T14:46:28.684032997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-72706aa0 10.129.0.78:7946"
ts=2026-04-23T14:46:35.895936157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47122"
level=debug ts=2026-04-23T14:46:45.301573641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:46:45.303269123Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:46:55.037119237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35008"
ts=2026-04-23T14:46:58.684960039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-72706aa0 10.129.0.78:7946"
ts=2026-04-23T14:47:15.902005224Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-72706aa0 (timeout reached)"
ts=2026-04-23T14:47:18.90157225Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-72706aa0 has failed, no acks received"
ts=2026-04-23T14:47:28.686987197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:47:35.901840078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45784"
ts=2026-04-23T14:47:35.901891961Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-72706aa0 (timeout reached)"
ts=2026-04-23T14:47:38.902079707Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-72706aa0 has failed, no acks received"
ts=2026-04-23T14:47:40.903140075Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-72706aa0 (timeout reached)"
ts=2026-04-23T14:47:43.90248443Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-72706aa0 has failed, no acks received"
level=debug ts=2026-04-23T14:47:45.958830755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:47:58.690063089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:47:59.600444421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58168"
ts=2026-04-23T14:47:59.609726379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58172"
level=debug ts=2026-04-23T14:48:23.060546278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:48:28.692725521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:48:35.908207885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58508"
ts=2026-04-23T14:48:58.615064512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34958"
ts=2026-04-23T14:48:58.694940908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
level=debug ts=2026-04-23T14:49:19.519227976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:49:20.050184733Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:49:20.052024813Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:49:20.053718053Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T14:49:20.053742186Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:49:28.616224463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45796"
ts=2026-04-23T14:49:28.696752598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:49:58.700182787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:50:21.607837129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:50:28.702085992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
ts=2026-04-23T14:50:35.920654735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54016"
ts=2026-04-23T14:50:47.823547602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41706"
ts=2026-04-23T14:50:58.703222621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
level=debug ts=2026-04-23T14:51:11.008298462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:51:11.010184195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:51:28.704057756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:51:35.926915428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38464"
ts=2026-04-23T14:51:58.706714515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:52:11.619953627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:52:17.82962157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36020"
ts=2026-04-23T14:52:28.70850587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"
ts=2026-04-23T14:52:35.933699672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47776"
ts=2026-04-23T14:52:58.711101682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
level=debug ts=2026-04-23T14:53:09.082909316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:53:28.635596708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44390"
ts=2026-04-23T14:53:28.713507138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
ts=2026-04-23T14:53:35.939992534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39614"
ts=2026-04-23T14:53:47.833460361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45520"
ts=2026-04-23T14:53:58.636959153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46698"
ts=2026-04-23T14:53:58.714837603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
level=debug ts=2026-04-23T14:54:01.322058475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:54:28.716112799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e609e32 10.129.0.78:7946"
ts=2026-04-23T14:54:35.945927806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35274"
level=debug ts=2026-04-23T14:54:36.329097586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T14:54:58.718708131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:55:05.902464644Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3e609e32 (timeout reached)"
ts=2026-04-23T14:55:05.948967401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47188"
ts=2026-04-23T14:55:08.902675592Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3e609e32 has failed, no acks received"
ts=2026-04-23T14:55:17.839745461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59432"
ts=2026-04-23T14:55:24.38426493Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3e609e32 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T14:55:25.902290725Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3e609e32 (timeout reached)"
ts=2026-04-23T14:55:28.722128745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6f92r-e5853ab1 10.130.0.83:7946"
ts=2026-04-23T14:55:28.901993155Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3e609e32 has failed, no acks received"
level=debug ts=2026-04-23T14:55:34.56873525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T14:55:34.570249702Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T14:55:35.951668263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55730"
ts=2026-04-23T14:55:47.840666708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59078"
ts=2026-04-23T14:55:58.72401324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-h4qjz-f6b90f2c 10.129.0.72:7946"