level=info ts=2026-04-21T16:00:53.503954496Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T16:00:53.503995109Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T16:00:53.504879698Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T16:00:53.505206683Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T16:00:53.505279852Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.79 inf=eth0
level=debug ts=2026-04-21T16:00:53.50623332Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T16:00:53.506297379Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.79 inf=eth0
level=info ts=2026-04-21T16:00:53.50636787Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-9ed63f52
ts=2026-04-21T16:00:53.506452922Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T16:00:53.506464028Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.79 advertisePort=7946
level=debug ts=2026-04-21T16:00:53.506479586Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.79 advertisePort=7946
level=info ts=2026-04-21T16:00:53.506578954Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T16:00:53.513019235Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T16:00:53.513041016Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T16:00:53.513051563Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.474569ms
level=error ts=2026-04-21T16:00:53.51305917Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T16:00:53.513070896Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T16:00:53.515181251Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T16:00:53.51520742Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T16:00:53.520053097Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T16:00:53.520119542Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-21T16:00:53.520153421Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T16:00:53.520156778Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T16:00:53.520128963Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T16:00:53.520236427Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-21T16:00:53.52025545Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T16:00:53.520285369Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T16:00:53.520421491Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T16:00:53.520428464Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T16:00:53.520470719Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T16:00:53.520492873Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T16:00:53.520510765Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T16:00:53.520632419Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T16:00:53.52064667Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T16:00:53.520673616Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T16:00:54.521455324Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T16:00:54.677764749Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T16:00:54.677830008Z caller=loki.go:599 msg="Loki started" startup_time=1.229843554s
ts=2026-04-21T16:00:55.341439275Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T16:00:55.34146913Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:00:58.617776111Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T16:00:58.617807631Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T16:00:59.678745898Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T16:00:59.678796399Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T16:01:03.324237661Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T16:01:03.324264128Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T16:01:10.09278311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:01:16.250333052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49138"
ts=2026-04-21T16:01:17.346330708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42716"
ts=2026-04-21T16:01:18.025357119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-04-21T16:01:18.02632927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.79:7946"
ts=2026-04-21T16:01:18.026355536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:51416"
ts=2026-04-21T16:01:18.027291277Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-04-21T16:01:18.028061124Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-04-21T16:01:18.0287901Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
ts=2026-04-21T16:01:18.028938041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
level=info ts=2026-04-21T16:01:18.029385772Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.516309789s
ts=2026-04-21T16:01:18.917192193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35136"
level=debug ts=2026-04-21T16:01:32.78919592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:01:40.773472459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60592"
ts=2026-04-21T16:01:52.067468161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:01:53.618426127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:02:00.84885732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56590"
level=debug ts=2026-04-21T16:02:13.547809971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:02:22.070048116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:02:30.851603567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36842"
ts=2026-04-21T16:02:40.776335981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48340"
level=debug ts=2026-04-21T16:02:42.820683675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T16:02:42.82229523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:02:52.071791313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:03:15.049830089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:03:22.073561885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:03:30.856094898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55952"
ts=2026-04-21T16:03:40.778892334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52002"
ts=2026-04-21T16:03:43.048713884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40638"
ts=2026-04-21T16:03:52.07498532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:04:11.805277962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:04:22.076443899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:04:40.782489219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44134"
ts=2026-04-21T16:04:52.077823223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:05:22.080006968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:05:26.825289848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:05:43.056069412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51422"
ts=2026-04-21T16:05:52.082398106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:06:00.867591182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51352"
ts=2026-04-21T16:06:13.057896793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50390"
ts=2026-04-21T16:06:22.084169991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:06:32.143002473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T16:06:43.059785967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55092"
ts=2026-04-21T16:06:52.085157954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:07:00.872812552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43336"
ts=2026-04-21T16:07:10.790500694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38276"
ts=2026-04-21T16:07:22.087374889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:07:30.061382403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T16:07:30.062949182Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:07:30.87527789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35328"
ts=2026-04-21T16:07:52.089178111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:08:13.064919538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38004"
ts=2026-04-21T16:08:22.090475196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:08:28.857868771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:08:52.09245526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:09:00.881131679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47372"
ts=2026-04-21T16:09:13.069166362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58894"
ts=2026-04-21T16:09:22.093496173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:09:24.326542758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:09:40.800047355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47222"
ts=2026-04-21T16:09:43.070929127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51194"
ts=2026-04-21T16:09:52.095614631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:10:13.227679778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:10:22.097332628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:10:30.887991652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40008"
ts=2026-04-21T16:10:43.07303991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50434"
ts=2026-04-21T16:10:52.098999883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=info ts=2026-04-21T16:10:59.678865797Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T16:10:59.682542791Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:10:59.684465881Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:10:59.686367829Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:10:59.686410114Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T16:10:59.686426678Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T16:11:00.889975758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41588"
level=debug ts=2026-04-21T16:11:03.104239337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:11:13.074159213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57316"
ts=2026-04-21T16:11:22.101364925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:11:40.806635585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36770"
ts=2026-04-21T16:11:52.102686296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:11:56.126822573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:11:56.128542782Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:12:13.077617345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38754"
ts=2026-04-21T16:12:22.103891769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:12:34.954126939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:12:40.81048228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43932"
ts=2026-04-21T16:12:52.105239372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:13:14.161500375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:13:22.107210014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:13:30.901526452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52092"
ts=2026-04-21T16:13:43.08258056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53824"
ts=2026-04-21T16:13:52.108841618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:13:53.898614769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:14:13.083581942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48260"
ts=2026-04-21T16:14:22.110293695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:14:30.906241983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42384"
ts=2026-04-21T16:14:40.815803169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43486"
ts=2026-04-21T16:14:43.085487532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52438"
level=debug ts=2026-04-21T16:14:48.194568252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:14:52.111547736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:15:22.113064074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:15:30.910890813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38698"
level=debug ts=2026-04-21T16:15:40.92110092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:15:40.922842505Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:15:43.088398381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42346"
ts=2026-04-21T16:15:52.11480567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:16:00.912503634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45618"
ts=2026-04-21T16:16:13.08944493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52508"
ts=2026-04-21T16:16:22.116583487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:16:28.658343281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:16:40.82329817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53880"
ts=2026-04-21T16:16:43.091018747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40420"
ts=2026-04-21T16:16:52.117770106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:17:15.599799924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:17:22.119716848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:17:43.093748049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44024"
ts=2026-04-21T16:17:52.121367684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:18:06.838065025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:18:13.094812888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52408"
ts=2026-04-21T16:18:22.123275769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:18:52.124405952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:19:01.395310746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:19:22.126272916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:19:30.928379315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36958"
ts=2026-04-21T16:19:40.834755125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54212"
ts=2026-04-21T16:19:52.127716537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:19:58.460701206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:19:58.462386384Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:20:00.930496441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46430"
ts=2026-04-21T16:20:13.100563649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46982"
ts=2026-04-21T16:20:22.129406222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:20:39.859660287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:20:43.101540296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42334"
ts=2026-04-21T16:20:52.131183815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=error ts=2026-04-21T16:20:59.693378236Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:20:59.695655342Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:20:59.697643571Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:20:59.697683369Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:21:10.838300727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42224"
ts=2026-04-21T16:21:13.102958109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42814"
ts=2026-04-21T16:21:22.132624216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:21:39.820447378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:21:52.133642815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:22:22.135003783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:22:30.94277282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60038"
level=debug ts=2026-04-21T16:22:39.227562661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:22:43.108522927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60164"
ts=2026-04-21T16:22:52.137505836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:23:20.988851125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:23:22.139451687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:23:43.112104411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35388"
ts=2026-04-21T16:23:52.141211395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:24:00.94918126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45424"
level=debug ts=2026-04-21T16:24:08.311731208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:24:08.313486239Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:24:10.848481065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40646"
ts=2026-04-21T16:24:13.113942076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42276"
ts=2026-04-21T16:24:22.142996303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:24:40.849970385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51066"
ts=2026-04-21T16:24:43.115381635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49660"
ts=2026-04-21T16:24:52.144841858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:24:54.554796806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:25:00.953986744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36588"
ts=2026-04-21T16:25:10.851659864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38940"
ts=2026-04-21T16:25:13.117180048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48224"
ts=2026-04-21T16:25:22.146322613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:25:40.853540033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44362"
ts=2026-04-21T16:25:52.147533861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:25:53.510588712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:26:00.958635619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37974"
ts=2026-04-21T16:26:13.120906372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54744"
ts=2026-04-21T16:26:22.14852091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:26:29.079598389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:26:40.856161929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57930"
ts=2026-04-21T16:26:52.150534684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:27:22.152598442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
level=debug ts=2026-04-21T16:27:23.067265452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:27:52.154075775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:28:00.966565662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39854"
level=debug ts=2026-04-21T16:28:07.1843424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:28:07.185952963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:28:10.860317188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35096"
ts=2026-04-21T16:28:13.128120965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42392"
ts=2026-04-21T16:28:22.1556666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ba468b73 10.129.0.78:7946"
ts=2026-04-21T16:28:43.129154287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50046"
ts=2026-04-21T16:28:45.838223777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48960"
ts=2026-04-21T16:28:45.845397475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48964"
ts=2026-04-21T16:28:52.157298886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:28:52.445375612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:28:55.508197749Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ba468b73 (timeout reached)"
ts=2026-04-21T16:28:58.507289204Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ba468b73 has failed, no acks received"
ts=2026-04-21T16:29:10.819828272Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ba468b73 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T16:29:13.131106443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35388"
ts=2026-04-21T16:29:22.158885085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:29:25.641081018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38832"
level=debug ts=2026-04-21T16:29:40.599713612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:29:43.132516725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59328"
ts=2026-04-21T16:29:52.160502387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:30:22.161994167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:30:30.978326778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44936"
level=debug ts=2026-04-21T16:30:39.980958713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:30:43.135465276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46964"
ts=2026-04-21T16:30:52.163389339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:30:55.645570936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36948"
level=error ts=2026-04-21T16:30:59.690383428Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:30:59.692162138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:30:59.693916255Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:30:59.693940987Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:31:13.136812602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46374"
ts=2026-04-21T16:31:22.164284703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4e75b4d2 10.129.0.78:7946"
ts=2026-04-21T16:31:30.983233385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58870"
level=debug ts=2026-04-21T16:31:40.181753824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:31:43.138465438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35254"
ts=2026-04-21T16:31:52.165947978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:31:55.648644247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53340"
ts=2026-04-21T16:32:22.168086723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:32:23.612957158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:32:23.614780589Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:32:52.170198684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:33:18.635050491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:33:22.17222406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:33:30.992637144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52162"
ts=2026-04-21T16:33:52.174329908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:33:55.656654505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60076"
ts=2026-04-21T16:34:00.994834397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46962"
level=debug ts=2026-04-21T16:34:03.578874723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:34:13.147415835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47364"
ts=2026-04-21T16:34:22.176339687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:34:30.998281829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52324"
ts=2026-04-21T16:34:43.148556464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44754"
level=debug ts=2026-04-21T16:34:43.981098858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:34:52.178029052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:34:55.660851495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58060"
ts=2026-04-21T16:35:01.000863268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51406"
level=debug ts=2026-04-21T16:35:20.797999347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:35:22.180348591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:35:52.181874281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:35:55.66502445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53666"
ts=2026-04-21T16:36:13.157464208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56878"
ts=2026-04-21T16:36:15.50749476Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4e75b4d2 (timeout reached)"
level=debug ts=2026-04-21T16:36:15.587829832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:36:15.589637752Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:36:18.507414689Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4e75b4d2 has failed, no acks received"
ts=2026-04-21T16:36:22.182717902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:36:31.103601128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59526"
ts=2026-04-21T16:36:31.111303792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59540"
ts=2026-04-21T16:36:35.508088846Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4e75b4d2 (timeout reached)"
ts=2026-04-21T16:36:38.508124954Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4e75b4d2 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T16:36:38.508193773Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4e75b4d2 has failed, no acks received"
ts=2026-04-21T16:36:43.158330754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47608"
ts=2026-04-21T16:36:52.18422652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:36:59.997892889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:37:22.186381162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
level=debug ts=2026-04-21T16:37:44.271599401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:37:48.500603506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49660"
ts=2026-04-21T16:37:52.188124001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:38:13.162185332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43506"
ts=2026-04-21T16:38:18.503251834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60628"
ts=2026-04-21T16:38:22.190123061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:38:41.569545314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:38:43.163722646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49366"
ts=2026-04-21T16:38:52.191669835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
level=debug ts=2026-04-21T16:39:16.745513515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:39:22.19282509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
ts=2026-04-21T16:39:43.167870822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40430"
ts=2026-04-21T16:39:52.194739092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:40:10.411278019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:40:10.413727345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:40:22.198940102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
ts=2026-04-21T16:40:31.029259724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41464"
ts=2026-04-21T16:40:43.171692461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44844"
level=debug ts=2026-04-21T16:40:49.501522884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:40:52.200481679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=error ts=2026-04-21T16:40:59.689914829Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:40:59.6916982Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:40:59.693468585Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:40:59.693492914Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:41:18.537793337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58668"
ts=2026-04-21T16:41:22.202486421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:41:31.033629426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51788"
level=debug ts=2026-04-21T16:41:37.733975443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:41:52.204342279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
ts=2026-04-21T16:42:22.205195785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:42:25.144514615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:42:52.206932294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:43:15.829519342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:43:22.208487023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:43:31.043302986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54852"
ts=2026-04-21T16:43:52.210585267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:44:13.18156715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55062"
level=debug ts=2026-04-21T16:44:17.309112268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:44:17.312282305Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:44:22.212116552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:44:43.184200424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47520"
ts=2026-04-21T16:44:52.214522008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-78f6f6ac 10.129.0.78:7946"
ts=2026-04-21T16:45:01.049774866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50646"
ts=2026-04-21T16:45:13.185608656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41846"
level=debug ts=2026-04-21T16:45:14.644725646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:45:18.550948388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59436"
ts=2026-04-21T16:45:22.218294078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:45:31.052466821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47536"
ts=2026-04-21T16:45:43.186588932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39020"
ts=2026-04-21T16:45:52.220528372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:46:13.989990713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:46:22.222008442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:46:50.323619077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:46:52.22359681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:47:22.224435671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:47:31.717641357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:47:52.226271251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:48:11.63705257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:48:11.638795842Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:48:13.195163175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36486"
ts=2026-04-21T16:48:13.451480355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34668"
ts=2026-04-21T16:48:13.458712889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34674"
ts=2026-04-21T16:48:22.228019419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:48:30.508356665Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-78f6f6ac (timeout reached)"
ts=2026-04-21T16:48:33.507411061Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-78f6f6ac has failed, no acks received"
ts=2026-04-21T16:48:38.419162978Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-78f6f6ac as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T16:48:49.36343973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39032"
ts=2026-04-21T16:48:52.229013093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:49:01.638473321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:49:13.198055792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51658"
ts=2026-04-21T16:49:22.230486984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:49:49.367240118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45050"
ts=2026-04-21T16:49:52.232074202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:50:01.272296505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:50:22.234347025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:50:31.075524404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43536"
ts=2026-04-21T16:50:49.369546376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58618"
level=debug ts=2026-04-21T16:50:50.380889668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:50:52.23554031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=error ts=2026-04-21T16:50:59.689724016Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:50:59.691720643Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:50:59.69327115Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T16:50:59.693304901Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:51:22.236388196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:51:26.53195811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:51:31.079900092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39870"
ts=2026-04-21T16:51:43.206443376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54580"
ts=2026-04-21T16:51:49.372544095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39826"
ts=2026-04-21T16:51:52.237349496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:52:02.532772873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:52:02.534247621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:52:19.373715444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58114"
ts=2026-04-21T16:52:22.23974178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:52:41.737409994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:52:49.375126415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47138"
ts=2026-04-21T16:52:52.241733817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:53:19.376383537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38112"
ts=2026-04-21T16:53:22.243694166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:53:34.089510344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:53:52.245222434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:54:01.092335001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55280"
ts=2026-04-21T16:54:22.246859203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:54:32.458458141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:54:49.381501025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53264"
ts=2026-04-21T16:54:52.247823868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:55:17.692186634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:55:22.249182501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:55:49.384492078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42106"
ts=2026-04-21T16:55:52.251605748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T16:55:53.210914837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:55:53.212509678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:56:22.253468124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:56:29.600027903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:56:52.255315587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T16:57:13.228392863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51286"
ts=2026-04-21T16:57:19.390137455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55742"
ts=2026-04-21T16:57:22.257193257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:57:31.150268318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:57:43.229410875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55292"
ts=2026-04-21T16:57:52.258080791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:58:19.679607297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:58:22.259483112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T16:58:52.260669418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T16:58:59.254588525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T16:59:13.234911273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53202"
ts=2026-04-21T16:59:22.262343574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T16:59:48.445996679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T16:59:48.447690985Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T16:59:52.263927113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
ts=2026-04-21T17:00:01.117678136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53198"
ts=2026-04-21T17:00:22.265909954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:00:43.24023655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39902"
level=debug ts=2026-04-21T17:00:48.151556995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:00:49.403704734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34794"
ts=2026-04-21T17:00:52.267568249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=error ts=2026-04-21T17:00:59.690520289Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:00:59.692269192Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:00:59.694001582Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:00:59.69402894Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:01:01.121443805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57680"
ts=2026-04-21T17:01:19.405045426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33410"
ts=2026-04-21T17:01:22.269195135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T17:01:31.917554874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:01:43.24343807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49678"
ts=2026-04-21T17:01:52.270361619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T17:02:11.368222588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:02:22.271474682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:02:31.127105028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55088"
ts=2026-04-21T17:02:52.27271136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T17:03:05.59655979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:03:19.410841685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46570"
ts=2026-04-21T17:03:22.273851562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
ts=2026-04-21T17:03:52.275232223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:04:08.029032141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:04:08.030662818Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:04:19.414493831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41768"
ts=2026-04-21T17:04:22.276018366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:04:52.277098071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:05:07.293674903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:05:13.257349311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55052"
ts=2026-04-21T17:05:19.418103797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60464"
ts=2026-04-21T17:05:22.278787868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:05:46.761419989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:05:49.419403113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57032"
ts=2026-04-21T17:05:52.281006279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:06:01.1428923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57924"
ts=2026-04-21T17:06:13.259217401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35144"
ts=2026-04-21T17:06:19.421064707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52120"
ts=2026-04-21T17:06:22.28320663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
level=debug ts=2026-04-21T17:06:30.0869226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:06:52.284697263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68200131 10.129.0.78:7946"
ts=2026-04-21T17:07:13.262019113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59152"
ts=2026-04-21T17:07:14.146226389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40400"
ts=2026-04-21T17:07:14.153259483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40408"
ts=2026-04-21T17:07:15.508048765Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-68200131 (timeout reached)"
ts=2026-04-21T17:07:18.508073555Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-68200131 has failed, no acks received"
level=debug ts=2026-04-21T17:07:20.597287851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:07:22.286834019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:07:31.149367637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46626"
ts=2026-04-21T17:07:52.288759535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:07:58.87196993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45098"
ts=2026-04-21T17:08:13.265270927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55110"
level=debug ts=2026-04-21T17:08:21.140061742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:08:21.141626131Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:08:22.29039921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
ts=2026-04-21T17:08:52.29213769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:09:13.268621602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50360"
ts=2026-04-21T17:09:22.294171324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T17:09:22.522478788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:09:28.877046627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39650"
ts=2026-04-21T17:09:52.295496217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:09:57.892908381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:09:58.878095542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56212"
ts=2026-04-21T17:10:13.27146265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55628"
ts=2026-04-21T17:10:22.297643662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:10:28.879629441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39374"
level=debug ts=2026-04-21T17:10:38.858231336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:10:52.300014699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=error ts=2026-04-21T17:10:59.689724623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:10:59.691702719Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:10:59.693370631Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:10:59.693393978Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:11:13.273927974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47730"
ts=2026-04-21T17:11:22.301389268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
level=debug ts=2026-04-21T17:11:35.220438178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:11:52.303091931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
ts=2026-04-21T17:12:01.170393029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46274"
ts=2026-04-21T17:12:13.276871512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55912"
ts=2026-04-21T17:12:22.304543991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
level=debug ts=2026-04-21T17:12:31.998409444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:12:32.00016857Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:12:52.306390447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:12:58.886852829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59090"
ts=2026-04-21T17:13:13.279443844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33800"
level=debug ts=2026-04-21T17:13:16.388677511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:13:22.308286332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
ts=2026-04-21T17:13:52.309500916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
level=debug ts=2026-04-21T17:14:10.281367597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:14:22.310903675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:14:31.180957091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50638"
level=debug ts=2026-04-21T17:14:46.788605026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:14:52.312272711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:14:58.893794253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39208"
ts=2026-04-21T17:15:01.182781322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60752"
ts=2026-04-21T17:15:22.314512968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T17:15:41.876345403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:15:52.31660987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:15:58.896443407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59016"
ts=2026-04-21T17:16:22.318099619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:16:31.187649958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40974"
level=debug ts=2026-04-21T17:16:33.630835706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:16:33.632489917Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:16:52.319436525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:17:01.18923916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40654"
level=debug ts=2026-04-21T17:17:08.451997473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:17:22.321263036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:17:43.642197466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:17:52.322398364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:17:58.902635889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58140"
ts=2026-04-21T17:18:22.323716697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:18:27.597359226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:18:31.195530262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43212"
ts=2026-04-21T17:18:43.299960874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36656"
ts=2026-04-21T17:18:52.325829189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:18:58.905871538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50524"
ts=2026-04-21T17:19:13.30097585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33618"
ts=2026-04-21T17:19:22.327406359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:19:27.811026772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:19:52.328521589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:20:22.330220849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
level=debug ts=2026-04-21T17:20:23.108420999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:20:23.109921988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:20:28.90930808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38258"
ts=2026-04-21T17:20:31.204382358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58800"
ts=2026-04-21T17:20:43.30621387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47404"
ts=2026-04-21T17:20:52.331222417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e9799d4a 10.129.0.78:7946"
level=error ts=2026-04-21T17:20:59.69008969Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:20:59.692092354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:20:59.693835852Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:20:59.693863115Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:21:02.924563987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39216"
ts=2026-04-21T17:21:02.93110218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39230"
ts=2026-04-21T17:21:13.307008525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49034"
level=debug ts=2026-04-21T17:21:18.081266461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:21:22.333318541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:21:35.507065284Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e9799d4a (timeout reached)"
ts=2026-04-21T17:21:37.983896353Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e9799d4a as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T17:21:38.507195674Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e9799d4a has failed, no acks received"
ts=2026-04-21T17:21:52.335006869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:22:04.396981303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:22:22.336710762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:22:52.338225999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T17:22:52.792594075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:22:58.571233746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57260"
ts=2026-04-21T17:23:13.31351392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58424"
ts=2026-04-21T17:23:22.340574117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:23:28.572291801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42038"
level=debug ts=2026-04-21T17:23:40.380672276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:23:52.342404037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:23:58.573550875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45860"
ts=2026-04-21T17:24:13.316043141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50222"
level=debug ts=2026-04-21T17:24:17.11534447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:24:17.116877111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:24:22.343936479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:24:28.574702911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58526"
ts=2026-04-21T17:24:43.317594406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58388"
ts=2026-04-21T17:24:52.345417122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:24:58.576175787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41882"
ts=2026-04-21T17:25:01.224972264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41334"
level=debug ts=2026-04-21T17:25:11.29067955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:25:13.318866476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57686"
ts=2026-04-21T17:25:22.346938707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:25:28.57752611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42874"
ts=2026-04-21T17:25:52.348711116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
level=debug ts=2026-04-21T17:26:03.462498689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:26:22.350521769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:26:31.232655777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58922"
level=debug ts=2026-04-21T17:26:48.321589011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:26:52.35162403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:27:22.353482501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:27:43.326179641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46542"
level=debug ts=2026-04-21T17:27:45.632802438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:27:52.355704881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:28:13.327507831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50740"
ts=2026-04-21T17:28:22.356808633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
level=debug ts=2026-04-21T17:28:22.414576407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:28:22.41626881Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:28:52.358103556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
level=debug ts=2026-04-21T17:29:22.273594234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:29:22.359190762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:29:31.246329476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57922"
ts=2026-04-21T17:29:52.361220181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T17:30:00.269306304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:30:22.363357955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:30:52.364855596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:30:53.953984787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:30:59.689513082Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:30:59.69107106Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:30:59.692603655Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:30:59.69262841Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:31:22.366561075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
level=debug ts=2026-04-21T17:31:51.217953001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:31:52.367829673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b22e4c02 10.129.0.78:7946"
ts=2026-04-21T17:32:01.258160339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50828"
ts=2026-04-21T17:32:13.343241882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58756"
ts=2026-04-21T17:32:22.369312266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
level=debug ts=2026-04-21T17:32:31.186669417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:32:31.188360154Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:32:52.371276062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"
ts=2026-04-21T17:33:01.262874822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60080"
level=debug ts=2026-04-21T17:33:11.768360737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:33:22.372641656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-msw2r-c0daf88a 10.128.0.73:7946"
ts=2026-04-21T17:33:31.265396125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37086"
ts=2026-04-21T17:33:43.347833628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40332"
level=debug ts=2026-04-21T17:33:48.677809328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:33:52.37469511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-5qqv9-2e22bbaf 10.129.0.76:7946"