level=info ts=2026-04-22T04:49:02.572063374Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T04:49:02.572088937Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T04:49:02.5732168Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T04:49:02.573494208Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:49:02.573567313Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=debug ts=2026-04-22T04:49:02.574352491Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:49:02.574418944Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=info ts=2026-04-22T04:49:02.574493158Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-cc3063cd
ts=2026-04-22T04:49:02.574598754Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T04:49:02.574614243Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=debug ts=2026-04-22T04:49:02.574628253Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=info ts=2026-04-22T04:49:02.574799759Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T04:49:02.582271691Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T04:49:02.582296134Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:49:02.582309061Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.511772ms
level=error ts=2026-04-22T04:49:02.582319301Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T04:49:02.582333926Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T04:49:02.586940115Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:49:02.586955874Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:49:02.589686898Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-22T04:49:02.589681323Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-22T04:49:02.589711367Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T04:49:02.58970564Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T04:49:02.589709704Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T04:49:02.58973997Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T04:49:02.589970688Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-22T04:49:02.589988022Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-22T04:49:02.589992121Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T04:49:02.58999542Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T04:49:02.590054782Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T04:49:02.590046387Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T04:49:02.590102835Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T04:49:02.59024948Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T04:49:02.590256252Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T04:49:02.590266166Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T04:49:03.590441311Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T04:49:03.786014317Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T04:49:03.786086083Z caller=loki.go:599 msg="Loki started" startup_time=1.277623856s
ts=2026-04-22T04:49:04.315650215Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:49:04.315674455Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:49:07.647142503Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:49:07.647169132Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T04:49:08.786687803Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T04:49:08.786747409Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T04:49:13.33658567Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:49:13.336617636Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:49:20.11945172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:49:24.783142519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52168"
ts=2026-04-22T04:49:24.783198465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.78:7946"
ts=2026-04-22T04:49:24.784184684Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.76:7946: dial tcp 10.130.0.76:7946: connect: connection refused"
level=info ts=2026-04-22T04:49:24.784202768Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.201862371s
ts=2026-04-22T04:49:26.267350366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35208"
ts=2026-04-22T04:49:32.344092213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40142"
ts=2026-04-22T04:49:33.372979699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43014"
level=debug ts=2026-04-22T04:49:36.824699339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:49:53.724503441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55848"
level=debug ts=2026-04-22T04:49:59.639476651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:50:00.092913199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T04:50:04.031251513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37526"
level=debug ts=2026-04-22T04:50:19.681295798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:50:30.096312169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T04:50:47.667867035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:50:47.669498065Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T04:51:00.099601343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:51:04.035257178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49888"
ts=2026-04-22T04:51:17.365323139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41596"
level=debug ts=2026-04-22T04:51:26.91982678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:51:30.1028086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T04:52:00.106232191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:52:30.109498524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T04:52:30.352976699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:52:47.370614052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39120"
ts=2026-04-22T04:52:53.736215075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35578"
ts=2026-04-22T04:53:00.112298076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T04:53:23.739672835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53912"
level=debug ts=2026-04-22T04:53:27.156034337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:53:30.115720284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:54:00.119363986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:54:04.044532237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53282"
ts=2026-04-22T04:54:23.744549004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36554"
ts=2026-04-22T04:54:30.122800906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T04:54:34.04796846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50342"
level=debug ts=2026-04-22T04:54:34.503567103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:54:53.747726114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43424"
ts=2026-04-22T04:55:00.125737046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T04:55:17.380863847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41354"
ts=2026-04-22T04:55:30.128822484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T04:55:47.017097538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:55:47.018819901Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T04:56:00.131923055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T04:56:30.135651663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:56:34.05490606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34118"
level=debug ts=2026-04-22T04:56:44.801973935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:56:47.386376104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52072"
ts=2026-04-22T04:56:53.755063925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59434"
ts=2026-04-22T04:57:00.138935108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:57:30.142361445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T04:57:34.06006014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35120"
level=debug ts=2026-04-22T04:57:43.67440544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:57:47.391462054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59352"
ts=2026-04-22T04:58:00.145668318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:58:17.394920944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43406"
ts=2026-04-22T04:58:23.762115352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56558"
ts=2026-04-22T04:58:30.148946651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T04:58:33.873195524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:59:00.152180046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:59:04.064999657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48506"
level=info ts=2026-04-22T04:59:08.787564437Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T04:59:08.795276383Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:59:08.798268185Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:59:08.800814842Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T04:59:08.800888831Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T04:59:08.800907034Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T04:59:17.399278285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:51018"
level=debug ts=2026-04-22T04:59:22.171043999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:59:23.766371665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33488"
ts=2026-04-22T04:59:30.155270615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T04:59:53.769386109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55778"
ts=2026-04-22T05:00:00.158593892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:00:15.73299715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:00:15.734817324Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:00:30.161840044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:01:00.165248534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:01:01.074583034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:01:04.07374331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57756"
ts=2026-04-22T05:01:30.168658729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:01:34.07692831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53008"
ts=2026-04-22T05:02:00.171672645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:02:02.505136075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:02:30.175082062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:02:34.081033739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58562"
ts=2026-04-22T05:02:53.779177111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41596"
level=debug ts=2026-04-22T05:02:55.017542101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:03:00.177904158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:03:04.084272348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49174"
ts=2026-04-22T05:03:17.411944389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59504"
ts=2026-04-22T05:03:23.782526904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54036"
ts=2026-04-22T05:03:30.181487892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:03:34.087287759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59652"
level=debug ts=2026-04-22T05:03:44.153749236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:03:53.786105209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58486"
ts=2026-04-22T05:04:00.18525652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:04:30.188553543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:04:46.380612449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:04:46.382432814Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:04:53.790201196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42212"
ts=2026-04-22T05:05:00.191253416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:05:17.420615387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46552"
ts=2026-04-22T05:05:30.194128114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:05:42.785752833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:06:00.197432774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:06:17.569037927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:06:30.201194156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:06:34.097386503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34342"
ts=2026-04-22T05:07:00.204443924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:07:08.38707285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:07:23.799862136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59032"
ts=2026-04-22T05:07:30.207300616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:07:53.54494687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:08:00.209996638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:08:04.102820053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56884"
ts=2026-04-22T05:08:17.429791288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35878"
level=debug ts=2026-04-22T05:08:28.537934693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:08:28.539733732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:08:30.213169418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:08:47.433234602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37212"
ts=2026-04-22T05:09:00.216512206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:09:06.619344252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:09:08.804475156Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:09:08.806493405Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:09:08.808237437Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:09:08.808268546Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:09:30.219858293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:09:53.811347627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34480"
level=debug ts=2026-04-22T05:09:56.45680734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:10:00.223131423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:10:30.226385529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:10:34.111507075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41238"
level=debug ts=2026-04-22T05:10:55.50860662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:11:00.229725952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:11:23.817894367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57356"
ts=2026-04-22T05:11:30.232547399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:11:32.145129071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:11:47.44467563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46532"
ts=2026-04-22T05:12:00.235312136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:12:04.116683513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54420"
ts=2026-04-22T05:12:30.23824966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:12:30.495485744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:12:30.497874401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:12:34.119922245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59758"
ts=2026-04-22T05:13:00.241375288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:13:05.518017989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:13:30.245201276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:13:34.124840724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49706"
level=debug ts=2026-04-22T05:13:40.809240542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:13:53.826873784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41416"
ts=2026-04-22T05:14:00.248958251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:14:17.454456418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45774"
level=debug ts=2026-04-22T05:14:28.028726348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:14:30.252571078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:14:53.831543486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41664"
ts=2026-04-22T05:15:00.262342452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:15:04.132088914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50300"
level=debug ts=2026-04-22T05:15:18.429378431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:15:23.835198123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34944"
ts=2026-04-22T05:15:30.265803041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:15:34.135371992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58984"
ts=2026-04-22T05:15:53.83882234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35348"
ts=2026-04-22T05:16:00.269827582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:16:14.855813855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:16:14.858917288Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:16:30.27314327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:17:00.276180759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:17:12.442856224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:17:30.279693724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:17:47.466620833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60374"
level=debug ts=2026-04-22T05:17:55.065104796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:18:00.283281794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:18:30.289932625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:18:31.33184163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:18:47.470388994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57652"
ts=2026-04-22T05:18:53.848474132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41534"
ts=2026-04-22T05:19:00.299044405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:19:07.614859497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:19:08.804824059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:19:08.807261953Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:19:08.809856338Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:19:08.809901935Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:19:30.302287864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:19:43.222063849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:19:43.224191325Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:20:00.308088613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:20:17.476446239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60508"
ts=2026-04-22T05:20:30.316999116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:20:38.647003757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:20:53.855248178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57440"
ts=2026-04-22T05:21:00.320523237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:21:30.323848153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:21:36.945828195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:22:00.327670602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:22:17.484643631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57526"
ts=2026-04-22T05:22:23.860926013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54058"
level=debug ts=2026-04-22T05:22:27.045088305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:22:30.330976051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:22:47.49103334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:54990"
ts=2026-04-22T05:22:53.864308433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:48834"
ts=2026-04-22T05:23:00.33413978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:23:04.156752669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55036"
level=debug ts=2026-04-22T05:23:18.925433485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:23:23.867502564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55850"
ts=2026-04-22T05:23:30.337553224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:23:34.160428086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45966"
ts=2026-04-22T05:23:53.871161105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56108"
ts=2026-04-22T05:24:00.341013011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:24:06.83342511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:24:06.83571664Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:24:30.344288781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:24:47.501863377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:51414"
ts=2026-04-22T05:24:53.882741806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36746"
ts=2026-04-22T05:25:00.347543334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:25:03.582087404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:25:30.350289681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
ts=2026-04-22T05:25:34.167471013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35688"
level=debug ts=2026-04-22T05:25:39.068551826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:26:00.353121619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6eef6a5e 10.129.0.69:7946"
level=debug ts=2026-04-22T05:26:20.693229533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:26:30.356605168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:26:47.510139803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33414"
ts=2026-04-22T05:26:53.894676408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35248"
ts=2026-04-22T05:27:00.360075665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:27:13.03657609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:27:17.457192182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44492"
ts=2026-04-22T05:27:17.465549178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44496"
ts=2026-04-22T05:27:17.513826999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:42420"
ts=2026-04-22T05:27:24.576725851Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6eef6a5e (timeout reached)"
ts=2026-04-22T05:27:27.576738454Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6eef6a5e has failed, no acks received"
ts=2026-04-22T05:27:30.362757873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
ts=2026-04-22T05:27:34.576806757Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6eef6a5e (timeout reached)"
ts=2026-04-22T05:27:37.576268747Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6eef6a5e has failed, no acks received"
ts=2026-04-22T05:27:47.516625319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57410"
ts=2026-04-22T05:28:00.365555495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
level=debug ts=2026-04-22T05:28:05.082308512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:28:05.084239391Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:28:23.899951342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39870"
ts=2026-04-22T05:28:26.147666545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34262"
ts=2026-04-22T05:28:30.368449774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:28:42.289709574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:29:00.37191455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=error ts=2026-04-22T05:29:08.805021101Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:29:08.807123396Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:29:08.808946951Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:29:08.808973341Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:29:17.52320447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38492"
ts=2026-04-22T05:29:30.374731499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:29:39.677930805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:29:47.526302507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:51098"
ts=2026-04-22T05:29:53.905687149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44804"
ts=2026-04-22T05:30:00.37785085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:30:23.909125272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40648"
ts=2026-04-22T05:30:30.381876486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:30:38.72621379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:30:56.156380205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48140"
ts=2026-04-22T05:31:00.390731412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:31:30.393742127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:31:38.600243334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:31:47.534027346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55090"
ts=2026-04-22T05:32:00.397526506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:32:17.537099782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45312"
level=debug ts=2026-04-22T05:32:20.463516681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:32:20.465073001Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:32:30.400687911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:32:47.540613122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47490"
ts=2026-04-22T05:33:00.403573932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
level=debug ts=2026-04-22T05:33:05.816944357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:33:26.165653345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56480"
ts=2026-04-22T05:33:30.406544588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
ts=2026-04-22T05:33:56.168867165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35168"
ts=2026-04-22T05:34:00.40934687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:34:07.945369694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:34:23.923924391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35790"
ts=2026-04-22T05:34:30.412274219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:34:59.50673271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:35:00.415705369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
ts=2026-04-22T05:35:17.549420878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57386"
ts=2026-04-22T05:35:30.419174293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
ts=2026-04-22T05:35:47.552581839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40306"
level=debug ts=2026-04-22T05:35:54.515120175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:36:00.422257789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:36:26.178319161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58424"
ts=2026-04-22T05:36:30.425107424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
level=debug ts=2026-04-22T05:36:34.918935348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:36:34.920716866Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:37:00.427792362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:37:17.557781501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:54314"
level=debug ts=2026-04-22T05:37:18.885052471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:37:30.431164748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:37:53.934549769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57328"
ts=2026-04-22T05:37:56.184275085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41114"
ts=2026-04-22T05:38:00.434570108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:38:14.385971755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:38:17.56281125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37928"
ts=2026-04-22T05:38:23.937356584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57180"
ts=2026-04-22T05:38:30.43792834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:38:49.804400297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:38:56.191114936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47290"
ts=2026-04-22T05:39:00.440417031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
level=error ts=2026-04-22T05:39:08.805092488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:39:08.806894353Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:39:08.808591873Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:39:08.808623782Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:39:30.443030478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
level=debug ts=2026-04-22T05:39:45.76479608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:40:00.446263207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:40:30.449694349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:40:33.525377322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:40:33.52769331Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:40:53.946200298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52716"
ts=2026-04-22T05:41:00.453025289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c21a3187 10.129.0.69:7946"
ts=2026-04-22T05:41:11.802133518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48956"
ts=2026-04-22T05:41:11.811896687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48970"
level=debug ts=2026-04-22T05:41:16.46570552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:41:30.455841231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:41:36.605746088Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c21a3187 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T05:41:53.951393347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42408"
level=debug ts=2026-04-22T05:41:58.193130246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:42:00.458729537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:42:30.462106588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T05:42:58.149622216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:43:00.465371171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:43:17.578555984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37454"
ts=2026-04-22T05:43:23.956372356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58250"
ts=2026-04-22T05:43:30.468141788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:43:40.875685813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:43:53.959413658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51508"
ts=2026-04-22T05:44:00.470909821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:44:23.962756774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59664"
level=debug ts=2026-04-22T05:44:27.805674979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:44:27.807539441Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:44:30.474250657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:45:00.477386087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:45:03.879244475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:45:17.58790168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43936"
ts=2026-04-22T05:45:30.480040699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:45:39.022895169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56666"
ts=2026-04-22T05:45:47.591133702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59206"
level=debug ts=2026-04-22T05:45:50.444518774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:46:00.483541448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:46:09.025733096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43582"
ts=2026-04-22T05:46:17.59435182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52884"
ts=2026-04-22T05:46:30.486307962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:46:46.001828756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:46:47.598707483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56942"
ts=2026-04-22T05:47:00.49267412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:47:09.030623406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58326"
ts=2026-04-22T05:47:23.971696154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39460"
ts=2026-04-22T05:47:30.497017581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:47:31.810005226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:48:00.500098245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:48:17.604912325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46352"
level=debug ts=2026-04-22T05:48:28.227311915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:48:28.228870895Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:48:30.502813499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:49:00.505904486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=error ts=2026-04-22T05:49:08.804610995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:49:08.806374079Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:49:08.807983088Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:49:08.808013641Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:49:10.468702104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:49:17.608877665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:34106"
ts=2026-04-22T05:49:30.508399331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:49:47.612399042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57922"
ts=2026-04-22T05:50:00.511213456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:50:07.795195265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:50:30.51389971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:50:49.047078464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:50:53.991218141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37844"
ts=2026-04-22T05:51:00.51714036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:51:17.617884216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40842"
ts=2026-04-22T05:51:30.520234702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:51:49.304430277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:52:00.52346448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:52:30.526194137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:52:39.49405686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:52:39.495737187Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:52:53.998270767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47882"
ts=2026-04-22T05:53:00.529577165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:53:09.047226633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51772"
ts=2026-04-22T05:53:30.533178703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:53:33.811033326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:53:54.002316849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50484"
ts=2026-04-22T05:54:00.536618923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:54:09.051924576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49462"
ts=2026-04-22T05:54:24.005694846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50062"
level=debug ts=2026-04-22T05:54:30.008497759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:54:30.540046654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:54:39.05497114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38108"
ts=2026-04-22T05:55:00.543201233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:55:24.010162404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51072"
level=debug ts=2026-04-22T05:55:25.282829523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:55:30.546599503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:55:59.838182419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:56:00.550170844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:56:09.060638783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40022"
ts=2026-04-22T05:56:24.014419997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38830"
ts=2026-04-22T05:56:30.553471649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T05:56:51.906597957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:56:51.908054876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T05:56:54.017361122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37554"
ts=2026-04-22T05:57:00.55633994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T05:57:30.559279811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:57:48.034442116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:58:00.561978866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
ts=2026-04-22T05:58:09.067394398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57350"
ts=2026-04-22T05:58:24.023225372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57616"
ts=2026-04-22T05:58:30.565236645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=debug ts=2026-04-22T05:58:43.922944687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:58:47.640625553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60818"
ts=2026-04-22T05:59:00.568846218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fb01cfc 10.129.0.69:7946"
level=error ts=2026-04-22T05:59:08.806561784Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:59:08.808490267Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:59:08.810252575Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T05:59:08.81027936Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:59:19.895637476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T05:59:30.572306471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T05:59:39.073747198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48038"
ts=2026-04-22T06:00:00.575728558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:00:18.782723493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:00:30.57956462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:00:33.351168634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43642"
ts=2026-04-22T06:00:33.360290279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43644"
ts=2026-04-22T06:00:39.576035205Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7fb01cfc (timeout reached)"
ts=2026-04-22T06:00:42.575349186Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7fb01cfc has failed, no acks received"
level=debug ts=2026-04-22T06:00:54.096877693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:00:54.098660634Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:00:54.575279195Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7fb01cfc (timeout reached)"
ts=2026-04-22T06:00:57.575646387Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7fb01cfc has failed, no acks received"
ts=2026-04-22T06:01:00.583143371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:01:17.649702627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33494"
ts=2026-04-22T06:01:30.586298399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:01:41.624063327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:01:54.035004261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43694"
ts=2026-04-22T06:02:00.589159257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:02:20.754095116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:02:30.592183483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:02:54.039470122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53766"
ts=2026-04-22T06:03:00.59557019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:03:13.754798176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:03:17.657872333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52508"
ts=2026-04-22T06:03:30.598656247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:04:00.601297151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:04:07.605674424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:04:17.662850289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59274"
ts=2026-04-22T06:04:24.045088762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57744"
ts=2026-04-22T06:04:30.603982742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:04:47.666477889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49734"
level=debug ts=2026-04-22T06:04:52.134038288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:04:52.135593835Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:05:00.606964669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
ts=2026-04-22T06:05:30.61019932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
level=debug ts=2026-04-22T06:05:36.308744864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:06:00.613884269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
level=debug ts=2026-04-22T06:06:29.275480176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:06:30.617647292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:06:47.674225667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53862"
ts=2026-04-22T06:07:00.620412143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:07:09.721613414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37396"
ts=2026-04-22T06:07:17.6777416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36262"
ts=2026-04-22T06:07:24.056863334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33022"
level=debug ts=2026-04-22T06:07:30.114346791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:07:30.624051092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:07:39.725086845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43050"
ts=2026-04-22T06:07:47.680809352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53634"
ts=2026-04-22T06:07:54.05992054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41094"
ts=2026-04-22T06:08:00.626744835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:08:17.683722973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36246"
ts=2026-04-22T06:08:24.063069171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59540"
ts=2026-04-22T06:08:30.629815464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:08:32.08104453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:08:54.066144403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44136"
ts=2026-04-22T06:09:00.63256817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
level=error ts=2026-04-22T06:09:08.805706181Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:09:08.807391298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:09:08.809167778Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:09:08.809199361Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T06:09:14.5164204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:09:14.518001441Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:09:17.689316551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52614"
ts=2026-04-22T06:09:24.068983805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46424"
ts=2026-04-22T06:09:30.635511197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
ts=2026-04-22T06:09:54.071490671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49202"
ts=2026-04-22T06:10:00.638613528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:10:09.73356738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58114"
level=debug ts=2026-04-22T06:10:11.371126213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:10:17.693404452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38446"
ts=2026-04-22T06:10:24.075213595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49210"
ts=2026-04-22T06:10:30.641893153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:10:49.831452766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:11:00.645118631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:11:09.737842225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60900"
ts=2026-04-22T06:11:17.698209369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33276"
ts=2026-04-22T06:11:30.64844323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:11:40.178653326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:12:00.65154969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:12:26.846577725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:12:30.655061264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
ts=2026-04-22T06:12:54.083748859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43470"
ts=2026-04-22T06:13:00.658195055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:13:08.52087828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:13:08.522572945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:13:09.745954626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50496"
ts=2026-04-22T06:13:30.661338356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:13:50.420158683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:14:00.664307613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
ts=2026-04-22T06:14:23.090393882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38130"
ts=2026-04-22T06:14:23.099779023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38146"
ts=2026-04-22T06:14:29.575664738Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4a5ac14e (timeout reached)"
ts=2026-04-22T06:14:30.667062249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4a5ac14e 10.129.0.69:7946"
ts=2026-04-22T06:14:32.57631606Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4a5ac14e has failed, no acks received"
level=debug ts=2026-04-22T06:14:43.637568993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:14:52.577239888Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4a5ac14e as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T06:14:54.57551548Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4a5ac14e (timeout reached)"
ts=2026-04-22T06:14:57.575142473Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4a5ac14e has failed, no acks received"
ts=2026-04-22T06:15:00.66961083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:15:18.976059009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:15:30.673119863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:15:36.991118342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58338"
ts=2026-04-22T06:16:00.676234102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:16:11.095344742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:16:30.679636465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:16:36.994524295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51876"
level=debug ts=2026-04-22T06:16:51.41275751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:16:51.414602073Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:16:54.094855753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57166"
ts=2026-04-22T06:17:00.68230843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:17:24.097272302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43274"
ts=2026-04-22T06:17:30.685072999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:17:44.330688246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:17:54.100930071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54488"
ts=2026-04-22T06:18:00.687560369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:18:17.718426355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43334"
ts=2026-04-22T06:18:24.104288508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43632"
ts=2026-04-22T06:18:30.691001738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:18:37.000982596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41296"
level=debug ts=2026-04-22T06:18:41.767786867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:18:54.107208287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49220"
ts=2026-04-22T06:19:00.693526536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=error ts=2026-04-22T06:19:08.806746127Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:19:08.808824348Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:19:08.810556907Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:19:08.810590754Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:19:30.696338408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:19:42.865669878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:19:47.724164622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:54912"
ts=2026-04-22T06:20:00.698871662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:20:07.005327249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57642"
ts=2026-04-22T06:20:17.72734201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36208"
ts=2026-04-22T06:20:30.701486562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:20:35.590827851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:21:00.70492306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:21:07.009702352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58562"
level=debug ts=2026-04-22T06:21:25.411349803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:21:25.413233773Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:21:30.708318887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:22:00.711216195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:22:07.014600996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38992"
level=debug ts=2026-04-22T06:22:16.943883276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:22:17.735467398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60720"
ts=2026-04-22T06:22:30.713864047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:23:00.716805034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:23:06.04066419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:23:30.719979899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:23:50.981817364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:24:00.722427446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:24:17.742523469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:42180"
ts=2026-04-22T06:24:24.124080465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:45054"
ts=2026-04-22T06:24:30.725223882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:24:31.745719192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:25:00.728296691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:25:17.747363034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57924"
ts=2026-04-22T06:25:24.128753003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36572"
ts=2026-04-22T06:25:30.731564458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:25:32.437559088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:25:32.439448839Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:25:54.131944489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40520"
ts=2026-04-22T06:26:00.734336016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
level=debug ts=2026-04-22T06:26:07.765519821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:26:24.13574016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50988"
ts=2026-04-22T06:26:30.737318406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:26:37.028493498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53792"
ts=2026-04-22T06:27:00.740757466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:27:03.902866357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:27:30.744029734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:27:37.032323294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48640"
level=debug ts=2026-04-22T06:27:51.825595109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:28:00.747548407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:28:24.143111753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59288"
ts=2026-04-22T06:28:30.750312235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:28:40.844620453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:29:00.753844415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=error ts=2026-04-22T06:29:08.805808116Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:29:08.807828594Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:29:08.809924971Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T06:29:08.809965571Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T06:29:20.412371232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:29:20.413829063Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:29:24.147364018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46888"
ts=2026-04-22T06:29:30.757351142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:29:56.660179921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:30:00.760856295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:30:17.762439034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49858"
ts=2026-04-22T06:30:24.151259612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55044"
ts=2026-04-22T06:30:30.763945761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-149d5438 10.129.0.69:7946"
ts=2026-04-22T06:30:47.765825922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:50910"
ts=2026-04-22T06:30:54.154182925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59652"
level=debug ts=2026-04-22T06:30:56.93371077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:31:00.76757852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:31:18.669010834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48322"
ts=2026-04-22T06:31:18.678897961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48338"
ts=2026-04-22T06:31:29.575141562Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-149d5438 (timeout reached)"
ts=2026-04-22T06:31:30.771058743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:31:32.575523539Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-149d5438 has failed, no acks received"
ts=2026-04-22T06:31:46.817914506Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-149d5438 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T06:31:47.769924894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:34984"
level=debug ts=2026-04-22T06:31:54.813438498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:32:00.774340341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
ts=2026-04-22T06:32:25.995547084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39950"
ts=2026-04-22T06:32:30.777807965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:32:38.435462442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:33:00.780376199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:33:17.249929673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:33:17.251454753Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:33:30.783093616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-748d22d0 10.129.0.69:7946"
ts=2026-04-22T06:33:56.001347825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60856"
ts=2026-04-22T06:34:00.786640506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:34:05.503636572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:34:24.166425278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53882"
ts=2026-04-22T06:34:30.789534825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
level=debug ts=2026-04-22T06:34:49.295691201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:35:00.79242917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:35:24.171234406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51784"
level=debug ts=2026-04-22T06:35:29.951709756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:35:30.795453576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:36:00.798136259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ftk8b-ade57d6e 10.129.0.66:7946"
level=debug ts=2026-04-22T06:36:22.490567091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:36:26.009753868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47776"
ts=2026-04-22T06:36:30.801081786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"
ts=2026-04-22T06:36:47.78601766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33022"
ts=2026-04-22T06:36:56.013427026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47676"
ts=2026-04-22T06:37:00.804002439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-748d22d0 10.129.0.69:7946"
level=debug ts=2026-04-22T06:37:16.356559383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T06:37:16.358137222Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T06:37:30.806880821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-748d22d0 10.129.0.69:7946"
ts=2026-04-22T06:37:47.790045816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56578"
ts=2026-04-22T06:37:54.179573478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39288"
ts=2026-04-22T06:38:00.809760594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-748d22d0 10.129.0.69:7946"
level=debug ts=2026-04-22T06:38:18.156557689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T06:38:30.812531652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g4pkw-57a83772 10.129.0.68:7946"