level=info ts=2026-05-07T14:59:40.684676866Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T14:59:40.684780769Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T14:59:40.686119391Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T14:59:40.686460141Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:59:40.686537955Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.70 inf=eth0
level=debug ts=2026-05-07T14:59:40.687469946Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:59:40.687540492Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.70 inf=eth0
level=info ts=2026-05-07T14:59:40.687568923Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-e581d435
ts=2026-05-07T14:59:40.687655697Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T14:59:40.687665791Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.70 advertisePort=7946
level=debug ts=2026-05-07T14:59:40.687681271Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.70 advertisePort=7946
level=info ts=2026-05-07T14:59:40.687816764Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T14:59:40.69445736Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.88:7946: dial tcp 10.128.0.88:7946: connect: connection refused"
ts=2026-05-07T14:59:40.695618075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.87:7946"
ts=2026-05-07T14:59:40.698378224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
level=info ts=2026-05-07T14:59:40.699651405Z caller=memberlist_client.go:728 msg="memberlist fast-join finished" joined_nodes=2 elapsed_time=11.837366ms
level=info ts=2026-05-07T14:59:40.699672437Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-07T14:59:40.701273835Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-07T14:59:40.701354853Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-07T14:59:40.701377484Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-07T14:59:40.701494557Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T14:59:40.701514622Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T14:59:40.701679727Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-07T14:59:40.701698408Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:59:40.701702338Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-07T14:59:40.701884141Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T14:59:40.701892695Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T14:59:40.701941983Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T14:59:40.701953856Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T14:59:40.701993814Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T14:59:40.702104694Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T14:59:40.702113954Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T14:59:40.702129836Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-07T14:59:40.702962821Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.88:7946: dial tcp 10.128.0.88:7946: connect: connection refused"
ts=2026-05-07T14:59:40.703562707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.87:7946"
ts=2026-05-07T14:59:40.705240826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
level=info ts=2026-05-07T14:59:40.706237204Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=6.557508ms
level=info ts=2026-05-07T14:59:41.702242695Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:59:41.702273225Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:59:41.702299719Z caller=loki.go:615 msg="Loki started" startup_time=1.076720596s
level=info ts=2026-05-07T14:59:46.703011764Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T14:59:46.703078873Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
level=debug ts=2026-05-07T14:59:57.731475189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:00:03.388674984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33748"
level=debug ts=2026-05-07T15:00:17.003715224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:00:22.210487823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:00:36.523510724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:37056"
level=debug ts=2026-05-07T15:00:37.734180688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:00:52.211448885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:01:02.983266814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:01:02.984445432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56956"
ts=2026-05-07T15:01:03.394357092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53722"
ts=2026-05-07T15:01:22.213047292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:01:30.70836875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:01:30.710210815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:01:52.214592778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:02:05.188928656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:02:06.529489809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53150"
ts=2026-05-07T15:02:22.217426389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:02:32.992176795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39672"
ts=2026-05-07T15:02:52.220231642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:03:20.450803329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:03:22.223140256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:03:32.998114663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37692"
ts=2026-05-07T15:03:52.226149659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:04:02.999794875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44972"
ts=2026-05-07T15:04:03.411903705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45048"
ts=2026-05-07T15:04:06.538060719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60692"
level=debug ts=2026-05-07T15:04:12.458296051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:04:22.228817627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:04:52.231006908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:05:03.004702138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52272"
ts=2026-05-07T15:05:06.542107448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45610"
ts=2026-05-07T15:05:22.233415465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:05:22.535732793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:05:52.235223251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:06:03.00847569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54524"
ts=2026-05-07T15:06:03.423398687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40410"
ts=2026-05-07T15:06:06.546392684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39040"
ts=2026-05-07T15:06:22.236530254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:06:31.647368098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:06:31.648957949Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:06:33.425862755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39796"
ts=2026-05-07T15:06:52.239542766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:07:06.54989735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34220"
level=debug ts=2026-05-07T15:07:12.719144679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:07:22.241970637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:07:52.244689445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:08:03.018554937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50360"
ts=2026-05-07T15:08:06.553346272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38446"
level=debug ts=2026-05-07T15:08:09.134170719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:08:22.246556534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:08:52.248030075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:08:54.947167755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:09:22.250026589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:09:33.444640042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45046"
level=info ts=2026-05-07T15:09:46.703323388Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T15:09:46.707245739Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:09:46.709301963Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:09:46.711138928Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:09:46.711165266Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:09:46.711177917Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T15:09:52.252721203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:09:56.752509722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:22.255480349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:10:33.031089449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:42524"
ts=2026-05-07T15:10:33.450215847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48002"
ts=2026-05-07T15:10:36.564661446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39658"
ts=2026-05-07T15:10:52.257287209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:10:53.360902133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:10:53.362439236Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:11:03.033049606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58560"
ts=2026-05-07T15:11:22.259889959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:11:28.040309701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:11:33.035021107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46496"
ts=2026-05-07T15:11:36.568142024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46722"
ts=2026-05-07T15:11:52.262769161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:12:03.457931406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48918"
ts=2026-05-07T15:12:22.264872584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:12:29.284732626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:12:52.266892563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:13:06.573604737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47054"
ts=2026-05-07T15:13:22.268708406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:13:24.582430119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:13:33.466234501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54444"
ts=2026-05-07T15:13:52.270277705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:14:02.479795552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:14:22.272124079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:14:33.471909484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50568"
ts=2026-05-07T15:14:36.580602816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57550"
level=debug ts=2026-05-07T15:14:39.476515334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:14:39.478624341Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:14:52.273936788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:15:22.276513007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:15:28.996150644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:15:33.477568164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49730"
ts=2026-05-07T15:15:36.585528576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57682"
ts=2026-05-07T15:15:52.278653724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:16:03.480809704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39090"
level=debug ts=2026-05-07T15:16:09.611117641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:16:22.280666227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:16:33.063871952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39876"
ts=2026-05-07T15:16:52.282913946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:17:06.59258117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56326"
level=debug ts=2026-05-07T15:17:09.611010353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:17:22.285128584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:17:36.594393012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:43182"
ts=2026-05-07T15:17:52.289376773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:18:10.206683054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:18:22.293464141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:18:33.073360754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44564"
ts=2026-05-07T15:18:33.495201647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57484"
ts=2026-05-07T15:18:36.598711359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46708"
level=debug ts=2026-05-07T15:18:46.788884655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:18:46.790636764Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:18:52.295636042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:19:03.501045733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35732"
ts=2026-05-07T15:19:22.29665724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:19:27.608270827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:19:46.718252793Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:19:46.720472108Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:19:46.722415488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:19:46.722441582Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:19:52.298769689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:20:03.50701718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57478"
ts=2026-05-07T15:20:06.60523383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60582"
level=debug ts=2026-05-07T15:20:07.311352649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:20:22.302029413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:20:52.304305151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:20:52.537286968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:21:22.306255742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:21:36.611862963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44006"
level=debug ts=2026-05-07T15:21:42.485116357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:21:52.308266933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:22:03.095500956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57208"
ts=2026-05-07T15:22:22.310435832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:22:36.017371197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:22:36.019778365Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:22:52.312788984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:23:06.618449608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45634"
level=debug ts=2026-05-07T15:23:10.956834039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:23:22.31751778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:23:33.10504921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47998"
ts=2026-05-07T15:23:33.529014428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60330"
ts=2026-05-07T15:23:36.620598922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45510"
ts=2026-05-07T15:23:52.323796458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:23:57.783246233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:24:03.107639539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49156"
ts=2026-05-07T15:24:03.532498427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48866"
ts=2026-05-07T15:24:22.32573702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:24:33.110315787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57814"
ts=2026-05-07T15:24:33.534765374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56486"
ts=2026-05-07T15:24:52.326986141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:24:57.218021766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:25:06.630490408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59888"
ts=2026-05-07T15:25:22.334466892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:25:36.6334714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:40452"
level=debug ts=2026-05-07T15:25:43.786654907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:25:52.340485706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:26:22.345722998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:26:26.983741155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:26:26.985382189Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:26:52.353525577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:27:03.554610167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43500"
ts=2026-05-07T15:27:22.35900936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:27:23.595857013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:27:33.557931189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43680"
ts=2026-05-07T15:27:52.361295915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:28:03.138376411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43804"
level=debug ts=2026-05-07T15:28:03.326557956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:28:22.363553141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:28:43.443941728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:28:52.365889749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:29:22.368632756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:29:30.554239265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:29:36.660621412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42816"
level=error ts=2026-05-07T15:29:46.716608123Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:29:46.71944045Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:29:46.72150498Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:29:46.721540615Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:29:52.371553493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:30:06.66172297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:51390"
level=debug ts=2026-05-07T15:30:18.387325238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:30:18.388959908Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:30:22.373742089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:30:52.376816245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:31:14.13711921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:31:22.37930207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:31:33.161726634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36970"
ts=2026-05-07T15:31:36.668398904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59236"
ts=2026-05-07T15:31:52.380726069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:31:58.78360904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:32:03.163682068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58624"
ts=2026-05-07T15:32:06.669995946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57034"
ts=2026-05-07T15:32:22.382524895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:32:33.837360928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:32:52.384505352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:33:22.38620146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:33:25.095139545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:33:33.170776942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47576"
ts=2026-05-07T15:33:52.387558655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:34:03.173356779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37292"
level=debug ts=2026-05-07T15:34:19.993092922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:34:19.994680593Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:34:22.389987706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:34:33.175394344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36756"
ts=2026-05-07T15:34:52.392025081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:35:03.600745208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60944"
level=debug ts=2026-05-07T15:35:14.368632277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:35:22.394606615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:35:49.140283633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:35:52.395910349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:36:03.60696428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52482"
ts=2026-05-07T15:36:06.688324642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:54368"
ts=2026-05-07T15:36:22.397048106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:36:35.460409122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:36:52.39865434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:37:03.190217149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59992"
ts=2026-05-07T15:37:03.612767021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46340"
ts=2026-05-07T15:37:06.691909969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:54486"
level=debug ts=2026-05-07T15:37:22.009522741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:37:22.40087895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:37:33.615297947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47618"
ts=2026-05-07T15:37:52.402379146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:38:06.695543261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35568"
level=debug ts=2026-05-07T15:38:07.262785088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:38:07.264641974Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:38:22.405042283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:38:33.621417586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38932"
ts=2026-05-07T15:38:36.696899972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45330"
ts=2026-05-07T15:38:52.407433733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:39:05.120900482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:39:22.410193785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:39:43.47571262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:39:46.714459542Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:39:46.716262486Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:39:46.718070141Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:39:46.718092553Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:39:52.412534215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:40:06.703519474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48982"
ts=2026-05-07T15:40:22.414991993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:40:36.705224015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59576"
level=debug ts=2026-05-07T15:40:41.570779269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:40:52.417304759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:41:03.635042488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57778"
ts=2026-05-07T15:41:06.706077873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48038"
ts=2026-05-07T15:41:22.419691671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:41:29.733242907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:41:36.707252763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55694"
ts=2026-05-07T15:41:52.42260928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:42:03.641803326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50996"
ts=2026-05-07T15:42:06.708615083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59176"
ts=2026-05-07T15:42:22.425569587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:42:24.247841133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:42:24.249416873Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:42:36.710542364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49216"
ts=2026-05-07T15:42:52.427955845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:43:03.647794584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38976"
level=debug ts=2026-05-07T15:43:11.958374296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:43:22.430996677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:43:52.433913362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:44:01.349996829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:44:03.653117448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54228"
ts=2026-05-07T15:44:22.435764295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:44:33.656363354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36668"
ts=2026-05-07T15:44:36.72023096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47268"
ts=2026-05-07T15:44:52.437825495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:44:57.001144591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:22.439069552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:45:36.723854817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56002"
level=debug ts=2026-05-07T15:45:50.506202959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:52.441051268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:46:03.665733638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34988"
ts=2026-05-07T15:46:22.444026985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:46:26.455602144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:46:26.457244599Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:46:52.446846587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:47:03.670793861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54526"
ts=2026-05-07T15:47:22.449066865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:47:26.474509296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:47:52.452107048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:48:03.677815457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58072"
ts=2026-05-07T15:48:22.454517853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:48:24.474056008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:48:33.68050103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35674"
ts=2026-05-07T15:48:52.45659829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:49:03.683915742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55388"
level=debug ts=2026-05-07T15:49:03.82929746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:49:22.457930431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:49:33.686765664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45062"
ts=2026-05-07T15:49:36.743616457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:37312"
level=error ts=2026-05-07T15:49:46.715363717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:49:46.717210053Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:49:46.719073573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:49:46.719103293Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:49:52.45989221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:50:01.291791717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:50:03.258148411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36540"
ts=2026-05-07T15:50:22.462447393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:50:33.69242015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38036"
ts=2026-05-07T15:50:36.74769987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34028"
ts=2026-05-07T15:50:52.464855205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:50:55.991927007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:50:55.993552697Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:51:22.467891686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:51:34.112096926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:51:36.75091226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45256"
ts=2026-05-07T15:51:52.470971699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:52:22.473319799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:52:22.64765207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:52:33.269035772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43734"
ts=2026-05-07T15:52:52.475345752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:53:21.828986069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:53:22.478431569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:53:33.273054099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47072"
ts=2026-05-07T15:53:33.711313292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60236"
ts=2026-05-07T15:53:52.480576024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:54:11.185755002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:54:22.482291423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T15:54:36.765770324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35328"
ts=2026-05-07T15:54:52.484556717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:54:53.037881794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:54:53.039658715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:55:03.719930037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45814"
ts=2026-05-07T15:55:06.766666877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47904"
ts=2026-05-07T15:55:22.486525359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:55:36.767902645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44698"
level=debug ts=2026-05-07T15:55:39.646417334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:55:52.488770198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T15:56:22.491247918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:56:30.483791691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:56:36.771733719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45334"
ts=2026-05-07T15:56:52.494128086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:57:22.495934052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T15:57:24.496480807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:57:52.497508104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:57:59.269133966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:58:06.778508583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47922"
ts=2026-05-07T15:58:22.499779573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T15:58:42.247402213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:58:42.249185218Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:58:52.502419287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T15:59:03.302471267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60608"
ts=2026-05-07T15:59:22.504505087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T15:59:23.723651062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:59:36.7855318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52816"
level=error ts=2026-05-07T15:59:46.715708135Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:59:46.717641603Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:59:46.719457249Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:59:46.719482948Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:59:52.506144297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:00:17.220812657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:00:22.508464009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:00:33.752510483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51216"
ts=2026-05-07T16:00:52.510547619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:01:03.755672526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40750"
level=debug ts=2026-05-07T16:01:18.80974572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:01:22.512004672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:01:33.759179975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52984"
ts=2026-05-07T16:01:52.514037319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:02:20.555241132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:02:22.516017527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:02:33.323472044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:40782"
ts=2026-05-07T16:02:52.518430248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:03:20.072095858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:03:20.073748523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:03:22.520834912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:03:33.328725713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49418"
ts=2026-05-07T16:03:33.771073372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44932"
ts=2026-05-07T16:03:52.523360769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:04:00.844037378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:04:03.774013914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47206"
ts=2026-05-07T16:04:22.525460473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:04:51.272933492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:04:52.527767359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:05:03.337572429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:40226"
ts=2026-05-07T16:05:03.780260918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49476"
ts=2026-05-07T16:05:06.812311431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58042"
ts=2026-05-07T16:05:22.530866205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:05:32.852818316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:05:52.533513361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:06:03.786023831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39742"
ts=2026-05-07T16:06:22.535752874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:06:26.455208305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:06:33.788797307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55662"
ts=2026-05-07T16:06:52.53824281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:07:03.346301872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49246"
level=debug ts=2026-05-07T16:07:09.982497696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:07:09.984001466Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:07:22.540880924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:07:36.823437609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39054"
ts=2026-05-07T16:07:52.543913064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:07:57.429025989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:08:03.351268651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58146"
ts=2026-05-07T16:08:22.546166805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:08:33.800210141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35788"
ts=2026-05-07T16:08:52.548530288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:08:56.403395528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:09:03.356770761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33846"
ts=2026-05-07T16:09:22.55043921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:09:31.609922309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:09:33.358365923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53426"
level=error ts=2026-05-07T16:09:46.715454388Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:09:46.717236566Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:09:46.718896689Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:09:46.718916981Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:09:52.552230905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:10:03.360252527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41330"
ts=2026-05-07T16:10:03.809905397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42046"
level=debug ts=2026-05-07T16:10:15.310452761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:10:22.554204977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:10:52.556753037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:11:03.366020366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59998"
level=debug ts=2026-05-07T16:11:03.461994482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:11:03.463738415Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:11:22.558150519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:11:33.368795562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53726"
level=debug ts=2026-05-07T16:11:48.359977884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:11:52.560464953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:12:22.562518829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:12:33.824443942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58858"
level=debug ts=2026-05-07T16:12:46.545225881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:12:52.564606727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:13:22.567139296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:13:33.378586652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39682"
level=debug ts=2026-05-07T16:13:48.384002364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:13:52.569012698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:14:03.832859373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39512"
ts=2026-05-07T16:14:06.85805271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33012"
ts=2026-05-07T16:14:22.57046904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:14:32.294558878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:14:52.571978758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:15:22.573533232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:15:26.357114844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:15:26.358796689Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:15:36.864849724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56990"
ts=2026-05-07T16:15:52.575655986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:16:03.391902372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:42844"
level=debug ts=2026-05-07T16:16:08.02669177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:16:22.577984798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:16:33.394594501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54644"
ts=2026-05-07T16:16:33.84680106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56492"
level=debug ts=2026-05-07T16:16:46.253507519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:16:52.579228812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:17:03.39669638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41870"
ts=2026-05-07T16:17:22.580721736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:17:33.399198632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37718"
ts=2026-05-07T16:17:36.874798535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:43880"
level=debug ts=2026-05-07T16:17:40.495182269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:17:52.582054035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:18:22.583934188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:18:31.01585816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:18:52.585838002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:19:22.588823599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:19:22.670682152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:19:22.672447366Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:19:33.86314829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44140"
ts=2026-05-07T16:19:36.884659202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56222"
level=error ts=2026-05-07T16:19:46.717247858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:19:46.719011482Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:19:46.720657171Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:19:46.720678061Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:19:52.591464312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:19:57.841592281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:20:06.886029024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57934"
ts=2026-05-07T16:20:22.594217018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:20:33.414892881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47584"
level=debug ts=2026-05-07T16:20:37.131465711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:20:52.59561603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:21:22.596865584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:21:33.419299201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36038"
level=debug ts=2026-05-07T16:21:34.191205164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:21:36.893137281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56744"
ts=2026-05-07T16:21:52.598094325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:22:03.421324167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46384"
ts=2026-05-07T16:22:06.894971094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53636"
ts=2026-05-07T16:22:22.600165551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:22:33.423909511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41626"
ts=2026-05-07T16:22:33.880781654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51170"
level=debug ts=2026-05-07T16:22:35.597701115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:22:36.895853409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47602"
ts=2026-05-07T16:22:52.602825816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:23:03.426271012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33168"
level=debug ts=2026-05-07T16:23:15.586014249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:23:15.587481034Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:23:22.605857047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:23:33.88695692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55308"
ts=2026-05-07T16:23:36.899630256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:36854"
ts=2026-05-07T16:23:52.608445606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:24:03.430631557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59258"
level=debug ts=2026-05-07T16:24:10.926368781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:24:22.611147354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:24:36.903533598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39152"
ts=2026-05-07T16:24:52.613465005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:24:58.315557577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:25:03.896124494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42116"
ts=2026-05-07T16:25:06.904728529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53288"
ts=2026-05-07T16:25:22.616488695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:25:33.437675323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55496"
level=debug ts=2026-05-07T16:25:51.664907269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:25:52.619769733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:26:03.439987648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54116"
ts=2026-05-07T16:26:03.902424855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37776"
ts=2026-05-07T16:26:22.622381616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:26:33.442049842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54470"
ts=2026-05-07T16:26:33.90520902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41942"
level=debug ts=2026-05-07T16:26:47.448645041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:26:52.625707895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:27:03.907999986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56902"
ts=2026-05-07T16:27:06.912151638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46714"
ts=2026-05-07T16:27:22.627637783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:27:36.913385258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59418"
level=debug ts=2026-05-07T16:27:43.178850955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:27:43.180514057Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:27:52.629259541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:28:22.631357619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:28:36.755625557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:28:52.633377449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:29:03.91919223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37442"
level=debug ts=2026-05-07T16:29:13.04350228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:29:22.635323193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:29:33.921913614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38726"
level=error ts=2026-05-07T16:29:46.715124924Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:29:46.716947582Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:29:46.718595649Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:29:46.718619212Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:29:52.637623124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:30:03.460042225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57356"
ts=2026-05-07T16:30:03.924603507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36440"
level=debug ts=2026-05-07T16:30:04.616775345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:30:22.639649253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:30:52.641687088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:30:54.871878456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:31:03.930288897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48898"
ts=2026-05-07T16:31:22.644751527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:31:32.736109636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:31:32.737565581Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:31:33.932894933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59546"
ts=2026-05-07T16:31:36.93289698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50302"
ts=2026-05-07T16:31:52.646603349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:32:03.469906452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44644"
ts=2026-05-07T16:32:06.934195777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47594"
level=debug ts=2026-05-07T16:32:12.982416728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:32:22.648547896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:32:33.471637757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53658"
ts=2026-05-07T16:32:36.935594393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57820"
level=debug ts=2026-05-07T16:32:50.306500743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:32:52.650555748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:33:03.47360731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51596"
ts=2026-05-07T16:33:03.941243864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50902"
ts=2026-05-07T16:33:06.937263022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46154"
ts=2026-05-07T16:33:22.652510118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
ts=2026-05-07T16:33:36.939201919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38086"
level=debug ts=2026-05-07T16:33:44.601090881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:33:52.654293056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:34:06.940759508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53858"
ts=2026-05-07T16:34:22.655982685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:34:35.878641747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:34:52.65790291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:35:06.94470658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41454"
ts=2026-05-07T16:35:22.659150586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
level=debug ts=2026-05-07T16:35:30.761743658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:35:30.763262006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:35:33.95609536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55868"
ts=2026-05-07T16:35:52.661223974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
level=debug ts=2026-05-07T16:36:09.228867466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:36:22.662667423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9c793fbd 10.130.0.71:7946"
ts=2026-05-07T16:36:33.490963608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:38678"
ts=2026-05-07T16:36:52.664485831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-fp7dn-6ba4ec92 10.129.0.73:7946"
level=debug ts=2026-05-07T16:36:53.761026712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:37:22.666795763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"
ts=2026-05-07T16:37:33.496168791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46952"
ts=2026-05-07T16:37:36.955043613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53368"
level=debug ts=2026-05-07T16:37:47.808388261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:37:52.669541618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w29sl-ed7e27fd 10.128.0.87:7946"