level=info ts=2026-04-17T15:03:43.517776945Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T15:03:43.517823345Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T15:03:43.518828156Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T15:03:43.519132136Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T15:03:43.519203619Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.83 inf=eth0
level=debug ts=2026-04-17T15:03:43.52013345Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T15:03:43.520196218Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.83 inf=eth0
level=info ts=2026-04-17T15:03:43.520286914Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-158f7475
ts=2026-04-17T15:03:43.52035503Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T15:03:43.520367894Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.83 advertisePort=7946
level=debug ts=2026-04-17T15:03:43.520383172Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.83 advertisePort=7946
level=info ts=2026-04-17T15:03:43.520483659Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T15:03:43.529504987Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T15:03:43.529529407Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T15:03:43.529543498Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.061727ms
level=error ts=2026-04-17T15:03:43.529553827Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T15:03:43.529568472Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T15:03:43.532485144Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T15:03:43.532505127Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T15:03:43.53513551Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-17T15:03:43.535131863Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-17T15:03:43.535160857Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-17T15:03:43.535186301Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-17T15:03:43.535226965Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-17T15:03:43.535160604Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-17T15:03:43.535421821Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-17T15:03:43.535480628Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-17T15:03:43.535485145Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-17T15:03:43.535488291Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T15:03:43.535521008Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T15:03:43.535549357Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T15:03:43.535570788Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T15:03:43.535718541Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T15:03:43.535727997Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T15:03:43.535747693Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T15:03:44.536567082Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-17T15:03:44.681759106Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T15:03:44.681831869Z caller=loki.go:599 msg="Loki started" startup_time=1.235612724s
ts=2026-04-17T15:03:45.172723073Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T15:03:45.172751289Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:03:48.074860567Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T15:03:48.07488815Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T15:03:49.682259255Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T15:03:49.682311863Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T15:03:52.921560601Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T15:03:52.921586904Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:04:05.178244214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33704"
ts=2026-04-17T15:04:05.532244767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-17T15:04:05.534888085Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
ts=2026-04-17T15:04:05.535831567Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.84:7946: dial tcp 10.128.0.84:7946: connect: connection refused"
ts=2026-04-17T15:04:05.536682419Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.79:7946: dial tcp 10.128.0.79:7946: connect: connection refused"
ts=2026-04-17T15:04:05.536800352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.83:7946"
ts=2026-04-17T15:04:05.536828063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42078"
ts=2026-04-17T15:04:05.537371062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
level=info ts=2026-04-17T15:04:05.537889001Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=22.008314286s
level=debug ts=2026-04-17T15:04:07.266593416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T15:04:23.853393665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:04:26.336534933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44156"
ts=2026-04-17T15:04:32.130289898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:04:45.172547579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:04:56.338302452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54030"
ts=2026-04-17T15:05:02.132573864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:05:05.164109592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:05:22.844543585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38190"
level=debug ts=2026-04-17T15:05:31.904564302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T15:05:31.906240328Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:05:32.134812651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:05:39.323498885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49536"
ts=2026-04-17T15:05:52.847571717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57842"
ts=2026-04-17T15:05:56.343760096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45272"
ts=2026-04-17T15:06:02.137452894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:06:16.578683919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:06:32.139687106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:06:52.854066058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55668"
ts=2026-04-17T15:06:56.347334255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:49538"
ts=2026-04-17T15:07:02.141117916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:07:08.900993596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:07:32.143163062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:07:39.332593421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51202"
ts=2026-04-17T15:07:56.351951685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60368"
ts=2026-04-17T15:08:02.144926738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:08:16.892700335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:08:26.35412264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42154"
ts=2026-04-17T15:08:32.147315984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:08:39.335869021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54742"
ts=2026-04-17T15:09:02.150040403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:09:17.77083743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T15:09:32.152841551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:09:39.340662708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38532"
ts=2026-04-17T15:09:56.361899124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33868"
ts=2026-04-17T15:10:02.155010078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:10:09.341458728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49246"
level=debug ts=2026-04-17T15:10:16.994776801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T15:10:16.99653555Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:10:26.363765376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53908"
ts=2026-04-17T15:10:32.15768288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:11:02.160075359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:11:19.151646422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:11:22.884489734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54362"
ts=2026-04-17T15:11:32.161819419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:11:52.887738236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39348"
ts=2026-04-17T15:12:02.163886093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:12:07.828723739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:12:09.352910954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50194"
ts=2026-04-17T15:12:22.890453742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40142"
ts=2026-04-17T15:12:32.166243207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:13:02.168679492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:13:08.749900136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:13:26.377966986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35830"
ts=2026-04-17T15:13:32.171813394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:13:39.357159553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42292"
level=info ts=2026-04-17T15:13:49.683359893Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-17T15:13:49.686957048Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:13:49.688920968Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:13:49.690857996Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:13:49.690889623Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T15:13:49.690901402Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-17T15:13:52.900476215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43980"
ts=2026-04-17T15:13:56.379642917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51264"
ts=2026-04-17T15:14:02.174556095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:14:10.385083954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:14:32.175596823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:14:52.907463891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46750"
ts=2026-04-17T15:14:56.382855977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34126"
ts=2026-04-17T15:15:02.177716743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:15:12.00135836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:15:12.003137324Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:15:22.910496246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45502"
ts=2026-04-17T15:15:26.384637019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58716"
ts=2026-04-17T15:15:32.179769557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:15:56.387105007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37352"
ts=2026-04-17T15:16:02.181944285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:16:09.862686811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:16:26.389029106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37424"
ts=2026-04-17T15:16:32.183542559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:16:39.371703174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33430"
ts=2026-04-17T15:16:52.920320368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48606"
level=debug ts=2026-04-17T15:16:57.289962222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:17:02.185731739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:17:32.187781676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:17:52.926977375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57442"
level=debug ts=2026-04-17T15:17:55.945869654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:17:56.39571083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58930"
ts=2026-04-17T15:18:02.189532552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:18:09.379124848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35012"
ts=2026-04-17T15:18:22.92976197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34876"
ts=2026-04-17T15:18:32.191479438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:18:33.804529092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:18:56.40101519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46292"
ts=2026-04-17T15:19:02.193578073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:19:21.408839547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:19:21.41061667Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:19:32.195940803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:20:02.198504479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:20:13.427266135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:20:26.408425377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42380"
ts=2026-04-17T15:20:32.200959919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:20:55.125776218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:21:02.20328978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:21:22.953089216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51706"
ts=2026-04-17T15:21:32.20615328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:21:49.404708824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:21:56.416929557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34176"
ts=2026-04-17T15:22:02.208544582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:22:32.211516208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:22:39.401382791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49166"
level=debug ts=2026-04-17T15:22:49.780373264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:23:02.214905243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:23:09.402920777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40128"
ts=2026-04-17T15:23:22.966947824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37256"
ts=2026-04-17T15:23:32.217150057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:23:37.295643943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:23:37.297519431Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:23:49.69530788Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:23:49.698014413Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:23:49.699871385Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:23:49.699897221Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:23:52.97019412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60498"
ts=2026-04-17T15:24:02.219395953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:24:12.781768554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:24:22.973365098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52044"
ts=2026-04-17T15:24:26.428336995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50450"
ts=2026-04-17T15:24:32.220437979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:24:57.852779521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:25:02.221864476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:25:09.411732197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53722"
ts=2026-04-17T15:25:32.223736545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:25:34.101198267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:25:39.412692099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36124"
ts=2026-04-17T15:25:52.983079166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36984"
ts=2026-04-17T15:26:02.227953853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:26:21.51239133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:26:22.986290208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35080"
ts=2026-04-17T15:26:32.230280284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:26:52.990192938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51270"
ts=2026-04-17T15:27:02.232421231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:27:14.676022274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:27:14.678198322Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:27:32.23448581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:27:55.929249698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:27:56.446325143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58858"
ts=2026-04-17T15:28:02.235997768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:28:09.424110949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36908"
ts=2026-04-17T15:28:26.448014954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56708"
ts=2026-04-17T15:28:32.237294491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:28:39.425479881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47120"
level=debug ts=2026-04-17T15:28:51.359658846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:28:56.449776026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41794"
ts=2026-04-17T15:29:02.238499698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:29:09.427106194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48500"
level=debug ts=2026-04-17T15:29:31.851577518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:29:32.240436929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:30:02.242129024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:30:09.432059392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48664"
level=debug ts=2026-04-17T15:30:12.273710042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:30:32.243659845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:30:39.432958032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40982"
level=debug ts=2026-04-17T15:30:54.765082032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:30:54.767081948Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:30:56.456799318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60544"
ts=2026-04-17T15:31:02.245104175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:31:23.020301588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42922"
ts=2026-04-17T15:31:26.458874988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59638"
ts=2026-04-17T15:31:32.246982596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:31:35.577197768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:31:56.460607761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58564"
ts=2026-04-17T15:32:02.2494201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:32:10.001336562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:32:32.250822837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:32:39.443815596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38194"
ts=2026-04-17T15:32:56.464375838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34424"
ts=2026-04-17T15:33:02.252361393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:33:03.436564006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:33:26.466593669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35008"
ts=2026-04-17T15:33:32.254974523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:33:39.200986224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:33:49.696663701Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:33:49.698744461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:33:49.700656778Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:33:49.700687109Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:34:02.257741825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:34:09.450038784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46214"
ts=2026-04-17T15:34:23.040577155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58746"
ts=2026-04-17T15:34:32.260053508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:34:34.027416562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:34:34.029132934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:34:53.043959039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49038"
ts=2026-04-17T15:35:02.262518156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:35:24.660660862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:35:32.265491563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:35:39.456292529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45210"
ts=2026-04-17T15:36:02.268064414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:36:23.147776749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:36:32.270501615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:36:39.459908764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55734"
ts=2026-04-17T15:37:02.272761356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:37:22.656030013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:37:26.48682177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56144"
ts=2026-04-17T15:37:32.275285171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:37:39.463605394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43556"
ts=2026-04-17T15:37:56.488339548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37966"
ts=2026-04-17T15:38:02.278185355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:38:09.464891567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48292"
level=debug ts=2026-04-17T15:38:10.498184474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:38:23.065136669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52926"
ts=2026-04-17T15:38:32.280686358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:38:51.964068821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:38:51.966077746Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:39:02.283692719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:39:32.286540258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:39:52.513077569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:39:53.074439298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50740"
ts=2026-04-17T15:40:02.289832057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:40:26.500848628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44242"
level=debug ts=2026-04-17T15:40:28.428263837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:40:32.292679393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:41:02.294888232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:41:09.482781072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41060"
ts=2026-04-17T15:41:23.082186625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39956"
level=debug ts=2026-04-17T15:41:26.453382311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:41:32.29728309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:41:53.085801336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39256"
ts=2026-04-17T15:41:56.508077603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50152"
ts=2026-04-17T15:42:02.300160339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:42:06.054848244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:42:23.089306326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58244"
ts=2026-04-17T15:42:32.303721303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:42:56.639546637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:42:56.64154539Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:43:02.306595064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:43:32.309790189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:43:43.171921485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:43:49.69429514Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:43:49.696135808Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:43:49.697833881Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:43:49.69786803Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:43:56.517505923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53584"
ts=2026-04-17T15:44:02.311925462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:44:20.241742997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:44:23.105934651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:56138"
ts=2026-04-17T15:44:32.31343724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:45:02.314966251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:45:09.502498271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37788"
level=debug ts=2026-04-17T15:45:15.433449793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:45:32.317675682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:45:56.526009138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56234"
ts=2026-04-17T15:46:02.321144824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:46:13.8440379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:46:26.528163938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41434"
ts=2026-04-17T15:46:32.322871644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:46:53.122163846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53488"
level=debug ts=2026-04-17T15:46:58.837386368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:46:58.839184894Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:47:02.324699591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:47:26.533711365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51390"
ts=2026-04-17T15:47:32.326837066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:47:39.516229064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59322"
level=debug ts=2026-04-17T15:48:00.413020257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:48:02.329331209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:48:26.538401768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56568"
ts=2026-04-17T15:48:32.332308808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:48:47.115390683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:48:53.135383448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41520"
ts=2026-04-17T15:49:02.334335823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:49:09.524780656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53998"
level=debug ts=2026-04-17T15:49:22.167576159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:49:23.139178957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44334"
ts=2026-04-17T15:49:32.336368957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:49:39.526416861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52148"
ts=2026-04-17T15:50:02.338672339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:50:17.019974286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:50:32.341400841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:50:39.530940828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46038"
ts=2026-04-17T15:51:02.344877118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:51:12.273972856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:51:12.275557086Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:51:23.152108589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52738"
ts=2026-04-17T15:51:26.551947171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53556"
ts=2026-04-17T15:51:32.347828072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:51:39.534436382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48652"
ts=2026-04-17T15:52:02.35047301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:52:06.432733732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:52:23.157917903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53740"
ts=2026-04-17T15:52:32.352611339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:52:54.941305739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:53:02.354598063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:53:32.356781956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
level=debug ts=2026-04-17T15:53:33.740677818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:53:39.544430846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50208"
level=error ts=2026-04-17T15:53:49.69727361Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:53:49.699256853Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:53:49.701161067Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T15:53:49.701187776Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:54:02.358721376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:54:23.170605724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39730"
level=debug ts=2026-04-17T15:54:23.542299674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:54:32.360374209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:54:39.548333484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36552"
ts=2026-04-17T15:55:02.362070302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T15:55:17.668572562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:55:17.670367346Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:55:32.364557048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:56:02.366283563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:56:10.664744536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:56:32.368384318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:56:39.558694054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34438"
level=debug ts=2026-04-17T15:56:52.448653706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:56:53.186500136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38072"
ts=2026-04-17T15:57:02.372195813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:57:09.560191088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52630"
level=debug ts=2026-04-17T15:57:27.472926151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:57:32.375848695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:57:39.561088896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36414"
ts=2026-04-17T15:57:53.192405702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47520"
ts=2026-04-17T15:58:02.379105457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8ggh-71ea23a8 10.130.0.72:7946"
ts=2026-04-17T15:58:09.56275021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39290"
ts=2026-04-17T15:58:23.19572987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47922"
level=debug ts=2026-04-17T15:58:27.617908495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T15:58:32.381705723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
ts=2026-04-17T15:58:56.589454737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35154"
ts=2026-04-17T15:59:02.383211842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
level=debug ts=2026-04-17T15:59:29.419854254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T15:59:29.421672442Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T15:59:32.384883237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-8mm5c-dc90c83a 10.128.0.82:7946"
ts=2026-04-17T15:59:39.570321038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53366"
ts=2026-04-17T16:00:02.386970522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-789c0e0a 10.129.0.71:7946"
level=debug ts=2026-04-17T16:00:06.653042644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"