level=info ts=2026-04-22T04:48:51.063207689Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T04:48:51.063231071Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T04:48:51.063966573Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T04:48:51.064339074Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:48:51.064423325Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.68 inf=eth0
level=debug ts=2026-04-22T04:48:51.065395622Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:48:51.065455288Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.68 inf=eth0
level=info ts=2026-04-22T04:48:51.065511089Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-8238fc8e
ts=2026-04-22T04:48:51.065622164Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T04:48:51.065639938Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.68 advertisePort=7946
level=debug ts=2026-04-22T04:48:51.065660732Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.68 advertisePort=7946
level=info ts=2026-04-22T04:48:51.065801813Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T04:48:51.070677961Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T04:48:51.070694952Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:48:51.070705708Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.905558ms
level=error ts=2026-04-22T04:48:51.070714304Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T04:48:51.070727657Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T04:48:51.073264412Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:51.073278139Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:48:51.079590824Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T04:48:51.079577837Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-22T04:48:51.079594874Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T04:48:51.079608049Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T04:48:51.079624405Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-22T04:48:51.079661785Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T04:48:51.079695365Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T04:48:51.079906081Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-22T04:48:51.079916735Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-22T04:48:51.07992054Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T04:48:51.079975497Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T04:48:51.079984078Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T04:48:51.080015937Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T04:48:51.080432305Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T04:48:51.080443777Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T04:48:51.080462342Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T04:48:52.080716638Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T04:48:52.281351252Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T04:48:52.281412178Z caller=loki.go:599 msg="Loki started" startup_time=1.298632793s
ts=2026-04-22T04:48:52.646619976Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:52.646644871Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:48:56.524435218Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:56.524460411Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T04:48:57.281638298Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T04:48:57.281693457Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T04:49:00.65677142Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:49:00.656795958Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:49:12.550811566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:49:12.643008199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-04-22T04:49:12.645671566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.71:7946"
ts=2026-04-22T04:49:12.646997811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.68:7946"
ts=2026-04-22T04:49:12.647015281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52252"
ts=2026-04-22T04:49:12.648194498Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.67:7946: dial tcp 10.129.0.67:7946: connect: connection refused"
ts=2026-04-22T04:49:12.648947163Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.70:7946: dial tcp 10.129.0.70:7946: connect: connection refused"
ts=2026-04-22T04:49:12.649700828Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.69:7946: dial tcp 10.129.0.69:7946: connect: connection refused"
level=info ts=2026-04-22T04:49:12.649714289Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.578980744s
level=debug ts=2026-04-22T04:49:28.078252572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:49:33.925597312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T04:49:45.248681449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:50:00.459876393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38928"
ts=2026-04-22T04:50:03.92731189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:50:10.577711882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:50:33.929794433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T04:50:36.307179577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:50:36.308900084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T04:50:46.158581062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57140"
ts=2026-04-22T04:50:59.372377343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45932"
ts=2026-04-22T04:51:03.932256356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:51:16.349102945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:51:33.934701131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T04:52:03.935882963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T04:52:15.94774972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:52:33.938259374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:53:00.474474817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:37446"
ts=2026-04-22T04:53:03.940777664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:53:27.517309222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:53:33.943837558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:53:46.17569184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33912"
ts=2026-04-22T04:54:00.479365929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57764"
ts=2026-04-22T04:54:03.946988744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:54:16.177038619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50384"
ts=2026-04-22T04:54:33.948777018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T04:54:39.229113854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:54:46.178094268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50244"
ts=2026-04-22T04:55:03.950392491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:55:30.485941455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55672"
ts=2026-04-22T04:55:33.952700976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T04:55:46.182152308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38884"
level=debug ts=2026-04-22T04:55:54.485174547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:55:54.486654127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T04:56:00.488857384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49986"
ts=2026-04-22T04:56:03.955486863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T04:56:16.183153875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38724"
level=debug ts=2026-04-22T04:56:30.390071862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:56:33.957979563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T04:57:00.497220595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48694"
ts=2026-04-22T04:57:03.960171623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:57:28.084609709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:57:29.405243688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53810"
ts=2026-04-22T04:57:30.500058032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:50894"
ts=2026-04-22T04:57:33.961648732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T04:58:00.503035932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52832"
level=debug ts=2026-04-22T04:58:03.645563132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:58:03.963546282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:58:29.41060823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43314"
ts=2026-04-22T04:58:33.965790686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=info ts=2026-04-22T04:58:57.281983772Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T04:58:57.284321497Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:58:57.286171573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:58:57.287908463Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:58:57.287945225Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T04:58:57.287981657Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-22T04:58:58.9920062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:59:00.508407549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55394"
ts=2026-04-22T04:59:03.968575889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T04:59:29.414828193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54898"
ts=2026-04-22T04:59:33.971478005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:59:51.452058211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T04:59:51.453672773Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:00:00.513098654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:33686"
ts=2026-04-22T05:00:03.973286267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:00:33.974068319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:00:46.236511075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:01:03.975129689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:01:16.211517557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53346"
ts=2026-04-22T05:01:29.424160745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49986"
ts=2026-04-22T05:01:30.520890936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:40972"
ts=2026-04-22T05:01:33.977510005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:01:36.015107892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:02:03.980080762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:02:16.215128307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40794"
ts=2026-04-22T05:02:30.524614374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60780"
level=debug ts=2026-04-22T05:02:33.878343587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:02:33.982575478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:02:59.43209863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53498"
ts=2026-04-22T05:03:03.98563597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:03:09.704265163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:03:30.530734113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:32942"
ts=2026-04-22T05:03:33.987945741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:04:03.99124402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:04:11.40072868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:04:11.402331031Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:04:33.993901093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:04:46.8399759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:04:59.44119503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47090"
ts=2026-04-22T05:05:03.996110265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:05:30.538620938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38304"
ts=2026-04-22T05:05:33.998546964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:05:42.876073545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:06:00.541649457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34846"
ts=2026-04-22T05:06:04.000938426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:06:30.544774362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41058"
ts=2026-04-22T05:06:34.003803092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:06:44.9794497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:06:59.448558166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45512"
ts=2026-04-22T05:07:00.547639435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34458"
ts=2026-04-22T05:07:04.006999288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:07:19.737323067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:07:29.451249804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45830"
ts=2026-04-22T05:07:30.549988245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48456"
ts=2026-04-22T05:07:34.009748456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:07:46.245072198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58020"
ts=2026-04-22T05:07:59.454284461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47506"
ts=2026-04-22T05:08:04.012965497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:08:07.868964449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:08:07.870464836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:08:16.246480091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43966"
ts=2026-04-22T05:08:34.014724514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=error ts=2026-04-22T05:08:57.291708652Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:08:57.29353869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:08:57.295231667Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:08:57.295259179Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:09:04.017286131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:09:08.427371065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:09:34.020322725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:09:53.708059312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:09:59.463905756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55820"
ts=2026-04-22T05:10:00.559627145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56900"
ts=2026-04-22T05:10:04.023535053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:10:16.256231453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51766"
ts=2026-04-22T05:10:34.026401737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:10:45.461890845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:11:00.563103671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:43772"
ts=2026-04-22T05:11:04.029688009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:11:34.032399705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:11:35.561945623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:12:04.034311305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:12:17.551933012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:12:17.55420216Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:12:34.036620824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:12:53.639995868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:13:00.57122654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34936"
ts=2026-04-22T05:13:04.039197384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:13:16.272297375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58670"
ts=2026-04-22T05:13:29.477663626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56584"
ts=2026-04-22T05:13:30.573997887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:33368"
level=debug ts=2026-04-22T05:13:32.98789235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:13:34.040632003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:13:46.273526328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59028"
ts=2026-04-22T05:13:59.480023746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34662"
ts=2026-04-22T05:14:04.04253146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:14:20.412669294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:14:29.48312939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48878"
ts=2026-04-22T05:14:30.579593076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48736"
ts=2026-04-22T05:14:34.044467345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:15:00.582720411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34000"
ts=2026-04-22T05:15:04.045634248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:15:08.090567928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:15:30.585318357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46434"
ts=2026-04-22T05:15:34.050043532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:15:59.338836681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:15:59.340317361Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:16:00.58780889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52636"
ts=2026-04-22T05:16:04.053417409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:16:16.286286257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33736"
ts=2026-04-22T05:16:34.056418907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:16:38.270411408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:16:46.288020388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41900"
ts=2026-04-22T05:16:59.495373368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:32844"
ts=2026-04-22T05:17:04.058629199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:17:29.498435096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45922"
ts=2026-04-22T05:17:34.060736577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:17:40.450094121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:17:46.291508638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58426"
ts=2026-04-22T05:18:00.597307985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:50872"
ts=2026-04-22T05:18:04.063020904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:18:16.293716086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42272"
level=debug ts=2026-04-22T05:18:18.701156128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:18:34.066146689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:18:46.296128097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37318"
level=error ts=2026-04-22T05:18:57.298186059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:18:57.301894425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:18:57.304681795Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:18:57.304723669Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:18:59.806542008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:19:04.068930457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:19:16.297210332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33098"
ts=2026-04-22T05:19:34.072047692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:19:51.189921242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:19:51.192092312Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:20:04.073999786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:20:16.300803868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53916"
ts=2026-04-22T05:20:29.515164162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52340"
ts=2026-04-22T05:20:34.076014475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:20:46.302275285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50880"
level=debug ts=2026-04-22T05:20:53.1055379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:21:04.078065221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:21:34.08104164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:21:52.3908787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:22:04.08417083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:22:30.616159688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38796"
ts=2026-04-22T05:22:34.086229165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:22:48.038233263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:23:00.619316786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56806"
ts=2026-04-22T05:23:04.088890566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:23:29.530798448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33372"
ts=2026-04-22T05:23:30.622719212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46104"
ts=2026-04-22T05:23:34.092254591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:23:49.606409851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:23:59.533657553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44788"
ts=2026-04-22T05:24:04.09372155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:24:34.096173351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:24:45.452317718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:24:45.454071163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:24:46.323873448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42506"
ts=2026-04-22T05:25:04.098555763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:25:16.324822619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54614"
ts=2026-04-22T05:25:34.100581649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:25:44.138552342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:25:46.326503843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39728"
ts=2026-04-22T05:26:04.10329996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:26:32.559285559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:26:34.10579716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:27:04.108824262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:27:11.698241604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:27:34.111704684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:27:59.548094852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60748"
ts=2026-04-22T05:28:04.113856896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:28:06.586886859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:28:16.339123328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60806"
ts=2026-04-22T05:28:29.550871953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33518"
ts=2026-04-22T05:28:34.116017346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:28:51.243125037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:28:51.244881195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:28:57.290735859Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:28:57.292497914Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:28:57.294171147Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:28:57.294224776Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:28:59.554149383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44348"
ts=2026-04-22T05:29:04.118963149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:29:34.120986229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:29:40.704686796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:30:00.649139904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:43768"
ts=2026-04-22T05:30:04.122608624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:30:16.34997185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52392"
level=debug ts=2026-04-22T05:30:31.056878102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:30:34.124298567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:31:04.126813205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:31:16.354622288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33962"
level=debug ts=2026-04-22T05:31:22.219288918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:31:30.657775692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45352"
ts=2026-04-22T05:31:34.129199878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:32:00.660750013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56248"
ts=2026-04-22T05:32:04.131377459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:32:16.358815837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50260"
level=debug ts=2026-04-22T05:32:17.814873798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:32:29.570879315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56502"
ts=2026-04-22T05:32:30.66328418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46134"
ts=2026-04-22T05:32:34.134168594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:32:59.580740868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:32:59.58244582Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:33:04.136733488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:33:16.363431138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54322"
ts=2026-04-22T05:33:34.138450715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:33:46.365110015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39120"
level=debug ts=2026-04-22T05:33:47.973840668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:34:00.669870684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:35344"
ts=2026-04-22T05:34:04.140812823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:34:34.143321029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:34:38.158761527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:35:00.674103958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:40830"
ts=2026-04-22T05:35:04.145800229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:35:15.639455994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:35:29.583150087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42948"
ts=2026-04-22T05:35:30.677278727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39034"
ts=2026-04-22T05:35:34.148385261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:35:56.021119113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:35:59.585646127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57282"
ts=2026-04-22T05:36:04.150606006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:36:16.378403096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51506"
ts=2026-04-22T05:36:34.152197832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:36:43.001710521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:36:43.003352449Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:36:59.591601362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55734"
ts=2026-04-22T05:37:04.154614799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:37:30.684705809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54428"
ts=2026-04-22T05:37:34.157793828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:37:44.606397754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:37:46.385020012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36278"
ts=2026-04-22T05:37:59.596350739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54588"
ts=2026-04-22T05:38:00.687543279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39324"
ts=2026-04-22T05:38:04.160867596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:38:16.386679349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53666"
ts=2026-04-22T05:38:29.599141329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48644"
ts=2026-04-22T05:38:34.164089916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:38:41.43211687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:38:57.293097198Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:38:57.295685995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:38:57.297829507Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:38:57.297859467Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:38:59.601792006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43650"
ts=2026-04-22T05:39:04.166329464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:39:16.391657494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48292"
level=debug ts=2026-04-22T05:39:28.94658955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:39:30.695805558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:37526"
ts=2026-04-22T05:39:34.167755806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:39:46.393099957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56984"
ts=2026-04-22T05:40:04.169670633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:40:06.794268837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:40:16.394243285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54532"
ts=2026-04-22T05:40:34.173193739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:40:46.395618776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53058"
level=debug ts=2026-04-22T05:41:03.752663594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:41:03.754324107Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:41:04.175786729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:41:34.177032728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:41:59.614240143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45712"
level=debug ts=2026-04-22T05:42:03.533542325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:42:04.179439935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:42:16.403092599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44522"
ts=2026-04-22T05:42:34.181102749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
level=debug ts=2026-04-22T05:43:02.324130081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:43:04.183209489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:43:16.407482885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59938"
ts=2026-04-22T05:43:30.712114411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41514"
ts=2026-04-22T05:43:34.186272942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:43:46.409148678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50202"
level=debug ts=2026-04-22T05:43:48.5103819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:44:04.189004742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:44:16.410899618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53140"
ts=2026-04-22T05:44:30.717975535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45956"
ts=2026-04-22T05:44:34.191967489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:44:46.951077837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:45:00.720870455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46374"
ts=2026-04-22T05:45:04.193995611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:45:34.196183242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:45:46.418268445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37398"
level=debug ts=2026-04-22T05:45:47.658887032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:45:47.660846935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:46:04.198667363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
ts=2026-04-22T05:46:30.7273304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58820"
ts=2026-04-22T05:46:34.201982824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:46:39.204638898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:47:04.204999501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:47:15.476065282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:47:16.424769848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42420"
ts=2026-04-22T05:47:34.207849591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
ts=2026-04-22T05:47:59.640040297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55720"
level=debug ts=2026-04-22T05:48:04.150212052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:48:04.209661981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"
ts=2026-04-22T05:48:34.211540159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:48:46.456746974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:48:57.293314649Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:48:57.295260694Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:48:57.297048147Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:48:57.297073381Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:48:59.646046179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52732"
ts=2026-04-22T05:49:04.213206213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bb3f9d9e 10.129.0.71:7946"