level=info ts=2026-04-19T19:19:03.008401929Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T19:19:03.008425601Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T19:19:03.010142866Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T19:19:03.010652618Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T19:19:03.010773479Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.68 inf=eth0
level=debug ts=2026-04-19T19:19:03.011415928Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=info ts=2026-04-19T19:19:03.01148472Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-8b34eb38
level=debug ts=2026-04-19T19:19:03.011500666Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.68 inf=eth0
ts=2026-04-19T19:19:03.011549586Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T19:19:03.011562017Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.68 advertisePort=7946
level=debug ts=2026-04-19T19:19:03.011578739Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.68 advertisePort=7946
level=info ts=2026-04-19T19:19:03.011682863Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T19:19:03.018453316Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:19:03.01847302Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:03.018484867Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.803189ms
level=error ts=2026-04-19T19:19:03.018494465Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T19:19:03.018510431Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T19:19:03.020859687Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:03.020878664Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T19:19:03.029787696Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-19T19:19:03.029794877Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-19T19:19:03.029789252Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-19T19:19:03.029824407Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-19T19:19:03.02982142Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T19:19:03.029857598Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-19T19:19:03.029957897Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-19T19:19:03.029976914Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T19:19:03.030127914Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-19T19:19:03.030137511Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T19:19:03.030195821Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T19:19:03.030192143Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T19:19:03.030214384Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T19:19:03.030355242Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T19:19:03.184427474Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T19:19:03.184485148Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T19:19:04.185167798Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T19:19:04.2864981Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T19:19:04.286540923Z caller=loki.go:599 msg="Loki started" startup_time=1.341921009s
ts=2026-04-19T19:19:04.828951619Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:04.828974787Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:19:06.976745371Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:06.976775954Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T19:19:09.287416272Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T19:19:09.287476195Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T19:19:12.61838395Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:12.618406564Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:19:21.054510049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:19:22.234051435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52712"
ts=2026-04-19T19:19:23.779707002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.65:7946"
ts=2026-04-19T19:19:23.780204815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.67:7946"
ts=2026-04-19T19:19:23.780625117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.68:7946"
ts=2026-04-19T19:19:23.780648295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59994"
level=info ts=2026-04-19T19:19:23.780869399Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=20.762353137s
ts=2026-04-19T19:19:27.840326661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:38636"
ts=2026-04-19T19:19:30.549604269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59834"
level=debug ts=2026-04-19T19:19:38.263437832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:19:39.10286342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45138"
level=debug ts=2026-04-19T19:19:59.107814685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:20:02.783024181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39988"
ts=2026-04-19T19:20:02.904982108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:20:20.632028384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:20:32.906463817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:20:46.11167797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:20:46.113243448Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:20:50.812236447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47630"
ts=2026-04-19T19:21:02.908129119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:21:32.785423619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:21:32.787341939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46944"
ts=2026-04-19T19:21:32.909203395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:22:02.910121212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:22:20.816498606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:44534"
ts=2026-04-19T19:22:32.911849151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:22:39.195047222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:22:50.818037871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38770"
ts=2026-04-19T19:23:02.790995809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54312"
ts=2026-04-19T19:23:02.91287144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:23:32.79250712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56552"
ts=2026-04-19T19:23:32.914485031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:23:39.114024839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53000"
level=debug ts=2026-04-19T19:23:42.46551041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:23:50.821284962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47760"
ts=2026-04-19T19:24:02.915250657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:24:32.916230897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:24:39.116531078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58856"
level=debug ts=2026-04-19T19:24:42.007989605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:25:02.795604812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54978"
ts=2026-04-19T19:25:02.917848017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:25:09.117751652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44124"
ts=2026-04-19T19:25:32.79673856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35042"
ts=2026-04-19T19:25:32.918765942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:25:39.119136743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:41158"
level=debug ts=2026-04-19T19:25:48.686695795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:25:48.688426565Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:26:02.920232867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:26:09.120241116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48338"
ts=2026-04-19T19:26:32.921768907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:26:39.121671509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35934"
level=debug ts=2026-04-19T19:26:40.376556363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:27:02.923120091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:27:19.315975787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:27:32.924231085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:28:02.92557995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:28:05.825573406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:28:32.927117656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:28:50.283936188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:29:02.805689148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44100"
ts=2026-04-19T19:29:02.928646677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=info ts=2026-04-19T19:29:09.287552435Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T19:29:09.290296573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:29:09.292052808Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:29:09.2935557Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:29:09.293584635Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:29:09.293595518Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T19:29:20.833937867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37448"
ts=2026-04-19T19:29:32.929930879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:29:39.127783349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59094"
level=debug ts=2026-04-19T19:29:47.407408452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:29:47.408933389Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:30:02.807997596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33436"
ts=2026-04-19T19:30:02.931408862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:30:29.923830184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:30:32.932371708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:31:02.93412831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:31:12.638019586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:31:32.93573859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:31:39.133081161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44338"
ts=2026-04-19T19:31:50.839373348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53842"
level=debug ts=2026-04-19T19:31:57.796596326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:32:02.93690888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:32:32.938111223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:32:39.13496297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60134"
level=debug ts=2026-04-19T19:32:58.330741745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:33:02.939748954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:33:09.136187357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33316"
ts=2026-04-19T19:33:20.84276306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52778"
ts=2026-04-19T19:33:32.940715454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:33:35.984360027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:33:35.985871787Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:34:02.942360664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:34:27.308279014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:34:32.943805007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:35:02.821454317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39384"
ts=2026-04-19T19:35:02.945426218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:35:09.14158063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57130"
level=debug ts=2026-04-19T19:35:18.736447374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:35:32.822478183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52206"
ts=2026-04-19T19:35:32.946623271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:36:02.948437232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:36:07.968784302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:36:32.949613512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:36:46.426769959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:37:02.951177229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:37:32.829259358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:45622"
ts=2026-04-19T19:37:32.952428958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:37:44.380734873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:37:44.383390681Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:37:50.855020269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:36036"
ts=2026-04-19T19:38:02.953767488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:38:26.916723145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:38:32.955268355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:38:50.85844171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:34748"
ts=2026-04-19T19:39:02.834012452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:60710"
ts=2026-04-19T19:39:02.956934047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=error ts=2026-04-19T19:39:09.2957481Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:39:09.297362815Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:39:09.299050444Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:39:09.299079689Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:39:11.290216496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:39:32.835256365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35038"
ts=2026-04-19T19:39:32.958105275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:39:39.155177849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43880"
ts=2026-04-19T19:39:50.860703283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:60658"
level=debug ts=2026-04-19T19:39:50.875847727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:40:02.959151449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:40:32.960084248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:40:44.33551309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:41:02.961210133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:41:09.158875649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:56496"
level=debug ts=2026-04-19T19:41:31.253515812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:41:31.255273244Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:41:32.96212086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:42:02.963708331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:42:09.161378384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:46094"
ts=2026-04-19T19:42:20.866577273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:40534"
level=debug ts=2026-04-19T19:42:23.561806069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:42:32.843275385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35436"
ts=2026-04-19T19:42:32.965426121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:42:50.868334029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56180"
ts=2026-04-19T19:43:02.84457675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54442"
ts=2026-04-19T19:43:02.966459552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:43:24.79737537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:43:32.967519945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:43:50.871266038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:42138"
ts=2026-04-19T19:44:02.847087794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54948"
ts=2026-04-19T19:44:02.968898824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:44:08.968268072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:44:09.167154654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33322"
ts=2026-04-19T19:44:32.970003203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:44:39.168427115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45244"
level=debug ts=2026-04-19T19:45:01.125944721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:45:02.970898009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:45:32.972230379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:45:35.247653191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:45:35.2493004Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:45:39.171496396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:46010"
ts=2026-04-19T19:45:50.876784445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33268"
ts=2026-04-19T19:46:02.853647577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43358"
ts=2026-04-19T19:46:02.973734312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:46:09.172968397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43022"
ts=2026-04-19T19:46:32.974876623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:46:33.287550968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:46:39.174183005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49050"
ts=2026-04-19T19:47:02.856126736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47704"
ts=2026-04-19T19:47:02.97599497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:47:23.251205999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:47:32.977224821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:48:02.859057095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39248"
ts=2026-04-19T19:48:02.978890028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:48:11.557365536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:48:32.980292373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:48:54.311880001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:49:02.981763808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=error ts=2026-04-19T19:49:09.298157362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:49:09.300209219Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:49:09.302352526Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:49:09.302377227Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:49:20.886258761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:51710"
ts=2026-04-19T19:49:32.982747977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:49:53.730137839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:49:53.731951682Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:50:02.983982344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:50:20.888349044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:42566"
ts=2026-04-19T19:50:32.985418253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:50:50.889767501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57722"
level=debug ts=2026-04-19T19:50:51.211573555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:51:02.98625836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:51:32.987261442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:51:48.036379335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:51:50.892790753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:60692"
ts=2026-04-19T19:52:02.869742859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40080"
ts=2026-04-19T19:52:02.988584036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:52:32.87120451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56746"
ts=2026-04-19T19:52:32.990080138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:52:36.898332818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:52:39.188463849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:38274"
ts=2026-04-19T19:52:50.894783506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33426"
ts=2026-04-19T19:53:02.872587012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:58664"
ts=2026-04-19T19:53:02.991445628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:53:32.993136757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:53:38.634504477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:53:39.191138004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45888"
ts=2026-04-19T19:53:50.897825409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35694"
ts=2026-04-19T19:54:02.994388439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:54:09.192640401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60228"
ts=2026-04-19T19:54:20.899168761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55282"
level=debug ts=2026-04-19T19:54:32.921160118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:54:32.92271002Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:54:32.996092085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:55:02.876938357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59718"
ts=2026-04-19T19:55:02.997836534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:55:20.901795039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45724"
level=debug ts=2026-04-19T19:55:24.290752158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:55:32.878120564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:60888"
ts=2026-04-19T19:55:32.999024473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:56:03.000479567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:56:09.545565363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:56:20.904610181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45456"
ts=2026-04-19T19:56:33.001219378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:56:39.19973705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53460"
level=debug ts=2026-04-19T19:56:47.890486856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:56:50.906426043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49300"
ts=2026-04-19T19:57:03.002768007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T19:57:33.004286313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:57:33.180577048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:57:50.908571603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35206"
ts=2026-04-19T19:58:03.005746658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:58:20.909435203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:34434"
ts=2026-04-19T19:58:32.886764374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39470"
ts=2026-04-19T19:58:33.007468313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T19:58:35.138171565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:58:35.139907207Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:58:39.205764594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47456"
ts=2026-04-19T19:59:03.00944755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=error ts=2026-04-19T19:59:09.297842003Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:59:09.299900166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:59:09.301824365Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:59:09.301850946Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:59:17.051126309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:59:20.912132685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59780"
ts=2026-04-19T19:59:32.889831649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34846"
ts=2026-04-19T19:59:33.010928038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:59:56.046105684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:00:03.012634082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T20:00:20.915044092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:46748"
ts=2026-04-19T20:00:33.014214297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T20:00:41.935294642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:00:50.916650256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57742"
ts=2026-04-19T20:01:02.894208203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36286"
ts=2026-04-19T20:01:03.015463593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:01:09.21383547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43266"
ts=2026-04-19T20:01:20.918100759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35372"
level=debug ts=2026-04-19T20:01:25.867443621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:01:32.895365268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49652"
ts=2026-04-19T20:01:33.01635076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T20:02:03.018167301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:02:09.216622242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55534"
ts=2026-04-19T20:02:20.921121361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56334"
level=debug ts=2026-04-19T20:02:22.176261386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:02:22.17804446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:02:33.01935872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:02:39.218275068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:39158"
ts=2026-04-19T20:02:50.92286947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:36918"
ts=2026-04-19T20:03:03.020844622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T20:03:08.595203666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:03:32.901648683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44398"
ts=2026-04-19T20:03:33.021628842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T20:03:54.453647946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:04:03.022249852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:04:33.023146202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T20:04:39.222893428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40474"
level=debug ts=2026-04-19T20:04:43.713559206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:04:50.927511924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53394"
ts=2026-04-19T20:05:03.024544067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:05:20.928426706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48434"
level=debug ts=2026-04-19T20:05:28.474624467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:05:33.026050232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:05:39.226041244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53890"
ts=2026-04-19T20:05:50.92997812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57112"
ts=2026-04-19T20:06:02.908463747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44696"
ts=2026-04-19T20:06:03.026931768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T20:06:29.268277426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:06:29.270180986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:06:32.909972215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46654"
ts=2026-04-19T20:06:33.027915042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:06:39.229011285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55904"
ts=2026-04-19T20:07:03.028724011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:07:09.230578229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40106"
level=debug ts=2026-04-19T20:07:25.647719368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:07:32.912160645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38232"
ts=2026-04-19T20:07:33.029634181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:07:50.93642168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54160"
ts=2026-04-19T20:08:03.031201327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T20:08:09.233482319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53024"
level=debug ts=2026-04-19T20:08:22.295829553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:08:33.032647018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
ts=2026-04-19T20:08:39.234253877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35988"
ts=2026-04-19T20:08:50.938979191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49668"
ts=2026-04-19T20:09:03.033245466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=error ts=2026-04-19T20:09:09.297927297Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:09:09.300060657Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:09:09.30182522Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:09:09.301854524Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T20:09:14.229359904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:09:33.034186913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:10:03.035211868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ec3bce6d 10.128.0.69:7946"
level=debug ts=2026-04-19T20:10:10.812815276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:10:32.92050647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40112"
ts=2026-04-19T20:10:33.036146958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T20:10:39.239912937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47246"