level=info ts=2026-04-22T13:45:05.777063739Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T13:45:05.777090764Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T13:45:05.77789016Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T13:45:05.778178251Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:45:05.778245417Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=debug ts=2026-04-22T13:45:05.779122686Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:45:05.779179491Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=info ts=2026-04-22T13:45:05.779312671Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c07d562a
ts=2026-04-22T13:45:05.779382493Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T13:45:05.779394864Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=debug ts=2026-04-22T13:45:05.779413132Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=info ts=2026-04-22T13:45:05.779520155Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T13:45:05.786624331Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:45:05.786643831Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:45:05.786654932Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.138637ms
level=error ts=2026-04-22T13:45:05.786661881Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T13:45:05.786669016Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T13:45:05.789198297Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:05.78922289Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:45:05.792721549Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-22T13:45:05.792774809Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T13:45:05.792787158Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T13:45:05.792793469Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T13:45:05.792821722Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T13:45:05.792833736Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T13:45:05.792924888Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-22T13:45:05.79293757Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T13:45:05.793107017Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-22T13:45:05.793117189Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T13:45:05.793174629Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T13:45:05.793177236Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T13:45:05.793195393Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T13:45:05.793333495Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T13:45:05.793340593Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T13:45:05.793356192Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T13:45:06.794065944Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T13:45:06.94736216Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T13:45:06.947414852Z caller=loki.go:599 msg="Loki started" startup_time=1.228497267s
ts=2026-04-22T13:45:07.642229503Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:07.64225621Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:45:10.453305369Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:10.453333891Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:45:11.948131001Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T13:45:11.948191712Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T13:45:16.718910493Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:16.718934356Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:45:21.376423205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:45:25.092272365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35948"
ts=2026-04-22T13:45:25.092327876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.93:7946"
level=info ts=2026-04-22T13:45:25.092594635Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=19.305921274s
ts=2026-04-22T13:45:30.815597727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37968"
ts=2026-04-22T13:45:32.012124441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:49398"
ts=2026-04-22T13:45:34.370715269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:52702"
level=debug ts=2026-04-22T13:45:40.029167376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:45:48.338578775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:45:53.196394629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:37178"
level=debug ts=2026-04-22T13:45:59.491905818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:46:18.341728287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:46:23.198745656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48920"
level=debug ts=2026-04-22T13:46:23.915762804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:46:48.345154895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:46:52.346894772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:46:52.348915568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:46:53.2019719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:39832"
ts=2026-04-22T13:46:53.366861232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:60880"
ts=2026-04-22T13:47:18.347459938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:47:23.368091005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:37152"
level=debug ts=2026-04-22T13:47:32.496275886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:47:36.113612868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39166"
ts=2026-04-22T13:47:48.349665575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:48:18.352365139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:48:46.951841378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:48:48.354778955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:48:53.211842594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38214"
ts=2026-04-22T13:49:18.356766306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:49:36.123480358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48594"
ts=2026-04-22T13:49:48.359274571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:49:58.719769901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:50:18.361410937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:50:48.363980871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:51:00.346899249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:51:18.366691088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:51:48.369909599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:52:06.135746237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52974"
level=debug ts=2026-04-22T13:52:13.608554689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:52:13.610324353Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:52:18.372616725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:52:48.374711534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:52:53.39621759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56502"
level=debug ts=2026-04-22T13:53:05.208211789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:53:18.377548344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:53:23.397558095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53164"
ts=2026-04-22T13:53:48.380605307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:53:48.387410004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:53:53.235619633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40544"
ts=2026-04-22T13:53:53.39914329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43112"
ts=2026-04-22T13:54:18.383597704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:54:23.238714915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41784"
ts=2026-04-22T13:54:48.385485068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T13:54:48.458779725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T13:55:11.9485331Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T13:55:11.954410819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:55:11.956701906Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:55:11.958904958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:55:11.958955495Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:55:11.958974446Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T13:55:18.386871261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:55:23.243084982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34040"
ts=2026-04-22T13:55:23.406084138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58580"
level=debug ts=2026-04-22T13:55:34.30890513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:55:48.38920657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:55:53.246313753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59110"
level=debug ts=2026-04-22T13:56:10.343940478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:56:10.345588054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:56:18.391899019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:56:23.249515092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59088"
ts=2026-04-22T13:56:36.158844635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56878"
ts=2026-04-22T13:56:48.394899296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T13:56:53.413592048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55752"
level=debug ts=2026-04-22T13:57:05.335905781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:57:18.397359943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T13:57:48.399664569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T13:58:03.199142483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:58:18.401923278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:58:46.074758461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:58:48.403819223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T13:58:53.42323519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57230"
ts=2026-04-22T13:59:18.404935322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T13:59:22.592217928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:59:48.405983479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T14:00:05.731405149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:00:05.733656519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:00:18.407760065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:00:23.269415993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48000"
ts=2026-04-22T14:00:36.178035382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58662"
ts=2026-04-22T14:00:48.410657387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:00:53.272337936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:53160"
level=debug ts=2026-04-22T14:00:58.994544876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:01:06.18139141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57552"
ts=2026-04-22T14:01:18.412663665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:01:23.275377481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:47452"
level=debug ts=2026-04-22T14:01:48.008632682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:01:48.414032316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:01:53.278934886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:44766"
ts=2026-04-22T14:02:18.416116972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:02:23.442266928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48044"
level=debug ts=2026-04-22T14:02:36.819885357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:02:48.418063865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:03:06.191763592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45662"
ts=2026-04-22T14:03:18.420063996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:03:36.910585907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:03:48.421993667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:03:53.287222452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:51358"
ts=2026-04-22T14:03:53.450483051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41694"
ts=2026-04-22T14:04:06.196094002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51016"
level=debug ts=2026-04-22T14:04:15.612071259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:04:15.613897614Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:04:18.424432459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:04:48.426753082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T14:04:59.594624002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:05:06.200898174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44710"
level=error ts=2026-04-22T14:05:11.962858254Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:05:11.964857319Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:05:11.966782882Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:05:11.966810197Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:05:18.429497264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:05:23.295724675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42170"
ts=2026-04-22T14:05:48.431926395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:05:53.297945905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42306"
ts=2026-04-22T14:05:53.460523504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48006"
level=debug ts=2026-04-22T14:06:01.970243844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:06:18.433387035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:06:23.300603301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48454"
ts=2026-04-22T14:06:23.461613521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38622"
ts=2026-04-22T14:06:36.208332411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54760"
level=debug ts=2026-04-22T14:06:40.366848237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:06:48.436130978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:07:18.439187491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:07:23.30501878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:37624"
ts=2026-04-22T14:07:23.466514734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41026"
ts=2026-04-22T14:07:36.214117803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48548"
level=debug ts=2026-04-22T14:07:38.453438001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:07:48.441911332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:07:53.308383786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59260"
ts=2026-04-22T14:08:06.216546686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59006"
ts=2026-04-22T14:08:18.445194227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:08:23.311595091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59418"
ts=2026-04-22T14:08:23.470820323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43680"
level=debug ts=2026-04-22T14:08:33.044154766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:08:33.046000239Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:08:36.219211162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34678"
ts=2026-04-22T14:08:48.447840998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:09:06.222357111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49512"
level=debug ts=2026-04-22T14:09:09.801488225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:09:18.450648067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:09:48.453647416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:10:08.894603172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:10:18.456312977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:10:23.480372991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47564"
ts=2026-04-22T14:10:36.228067641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59440"
level=debug ts=2026-04-22T14:10:46.893231994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:10:48.458831948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:11:18.461304629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:11:23.437178038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:11:23.484169536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39728"
ts=2026-04-22T14:11:36.232306115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60960"
ts=2026-04-22T14:11:48.463871686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:12:06.235503622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47492"
level=debug ts=2026-04-22T14:12:09.812870345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:12:09.814700251Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:12:18.466853344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:12:23.329414597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:58246"
ts=2026-04-22T14:12:23.491388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34094"
ts=2026-04-22T14:12:36.238841192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46624"
ts=2026-04-22T14:12:48.469057938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
level=debug ts=2026-04-22T14:12:52.257017548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:13:18.470766279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:13:23.335181797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34478"
ts=2026-04-22T14:13:36.244377177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55214"
ts=2026-04-22T14:13:48.473016496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:13:52.078215755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:14:18.476047551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:14:26.87418521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:14:36.248936688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37340"
ts=2026-04-22T14:14:48.478686375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:14:53.504492854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39244"
ts=2026-04-22T14:15:06.251980057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56942"
level=error ts=2026-04-22T14:15:11.964352959Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:15:11.967546406Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:15:11.969377844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:15:11.969414623Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:15:18.481290836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:15:22.868093547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:15:48.484930054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:16:06.256434866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33106"
level=debug ts=2026-04-22T14:16:06.904082497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:16:06.906194747Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:16:18.488065186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:16:23.51082309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38202"
level=debug ts=2026-04-22T14:16:41.546413367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:16:48.490211295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:17:18.492903896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:17:23.354633614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40840"
ts=2026-04-22T14:17:23.515708043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52940"
level=debug ts=2026-04-22T14:17:31.846045058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:17:48.49555381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:17:53.517353469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56956"
level=debug ts=2026-04-22T14:18:15.997344614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:18:18.497798595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:18:23.358162484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:39550"
ts=2026-04-22T14:18:23.518941056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:33858"
ts=2026-04-22T14:18:48.499636959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:18:53.36095149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59328"
ts=2026-04-22T14:18:53.520796024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58216"
ts=2026-04-22T14:19:06.270122069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41240"
level=debug ts=2026-04-22T14:19:13.856052911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:19:18.501967733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:19:48.504324318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:19:53.364816595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48558"
level=debug ts=2026-04-22T14:19:57.310735945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:19:57.312494155Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:20:18.506318015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:20:23.367320636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:56486"
ts=2026-04-22T14:20:23.527599832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51248"
ts=2026-04-22T14:20:48.508424824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:20:53.3701187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34960"
level=debug ts=2026-04-22T14:20:57.594667373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:21:18.509644182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:21:23.372817942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34916"
ts=2026-04-22T14:21:23.532853956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:59704"
ts=2026-04-22T14:21:36.285403721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47254"
ts=2026-04-22T14:21:48.51153997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:21:53.533996219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48782"
level=debug ts=2026-04-22T14:21:58.34339292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:22:18.514612067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:22:32.924917967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:22:48.518032358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:22:53.391904369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57604"
ts=2026-04-22T14:22:53.538526251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:35844"
level=debug ts=2026-04-22T14:23:16.812848795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:23:18.521104855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:23:36.298724427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37912"
ts=2026-04-22T14:23:48.523990679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:23:53.398707687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42152"
level=debug ts=2026-04-22T14:23:54.021065318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:23:54.02268242Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:24:06.302267523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44316"
ts=2026-04-22T14:24:18.526420611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7442c1 10.128.0.92:7946"
ts=2026-04-22T14:24:23.401300565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:37128"
ts=2026-04-22T14:24:23.546076217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39994"
level=debug ts=2026-04-22T14:24:47.257091339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:24:48.529182809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
ts=2026-04-22T14:25:06.307244363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33084"
level=error ts=2026-04-22T14:25:11.962966721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:25:11.964658445Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:25:11.966433915Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:25:11.96645968Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:25:18.532203604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:25:47.721697259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:25:48.535060487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:26:18.538478958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:26:23.556271271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58248"
level=debug ts=2026-04-22T14:26:25.332361756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:26:48.541795509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
ts=2026-04-22T14:26:53.412959178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:53214"