level=info ts=2026-05-09T22:51:43.969757292Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-09T22:51:43.969800135Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-09T22:51:43.971691707Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-09T22:51:43.972015772Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T22:51:43.972084766Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.75 inf=eth0
level=debug ts=2026-05-09T22:51:43.973005133Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T22:51:43.973095619Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.75 inf=eth0
level=info ts=2026-05-09T22:51:43.973182534Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-169e5eba
ts=2026-05-09T22:51:43.973257052Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-09T22:51:43.973269199Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.75 advertisePort=7946
level=debug ts=2026-05-09T22:51:43.973289577Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.75 advertisePort=7946
level=info ts=2026-05-09T22:51:43.973426797Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-09T22:51:43.981194576Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T22:51:43.981219299Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T22:51:43.981234263Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.809415ms
level=error ts=2026-05-09T22:51:43.981246629Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-09T22:51:43.981261256Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-09T22:51:43.984955121Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T22:51:43.984977733Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T22:51:43.990046655Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-09T22:51:43.990119618Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-09T22:51:43.990141563Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-09T22:51:43.990154018Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-09T22:51:43.990150402Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-09T22:51:43.990200584Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-09T22:51:43.99030418Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-09T22:51:43.99030927Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-09T22:51:43.990468257Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-09T22:51:43.990479179Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-09T22:51:43.990541187Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-09T22:51:43.990535886Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-09T22:51:43.990558226Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-09T22:51:43.990707999Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-09T22:51:43.990717038Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-09T22:51:43.990729188Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-09T22:51:44.991843172Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-09T22:51:45.145177187Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-09T22:51:45.145241146Z caller=loki.go:615 msg="Loki started" startup_time=1.243376205s
ts=2026-05-09T22:51:45.587490429Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T22:51:45.587515992Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:51:48.428293192Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T22:51:48.428324097Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T22:51:50.145471012Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-09T22:51:50.145542553Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-09T22:51:55.855620829Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T22:51:55.855645645Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T22:52:05.609731567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:52:10.355792922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.75:7946"
ts=2026-05-09T22:52:10.355908561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33410"
ts=2026-05-09T22:52:10.357224514Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-05-09T22:52:10.358478365Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
level=info ts=2026-05-09T22:52:10.35848731Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=26.37721987s
ts=2026-05-09T22:52:11.033528818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52620"
ts=2026-05-09T22:52:11.460128409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54924"
ts=2026-05-09T22:52:12.416333621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:34792"
ts=2026-05-09T22:52:19.490452891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T22:52:23.128557445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T22:52:38.354038263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:52:49.49326993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T22:52:59.59560547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:53:19.495931374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T22:53:27.002648844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T22:53:27.004718856Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T22:53:45.531173569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37614"
ts=2026-05-09T22:53:49.498593559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T22:54:06.327961604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:54:12.252895139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:47720"
ts=2026-05-09T22:54:15.533749896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55956"
ts=2026-05-09T22:54:19.500844807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T22:54:49.502882725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T22:54:57.486901284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:55:19.505794439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
ts=2026-05-09T22:55:49.50887475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T22:56:11.308380652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:56:19.511617714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
ts=2026-05-09T22:56:49.515054837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T22:57:07.509352904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T22:57:15.550153697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39490"
ts=2026-05-09T22:57:19.518443711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
ts=2026-05-09T22:57:49.521024051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T22:57:59.851304136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T22:57:59.852875955Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T22:58:12.266656083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:46048"
ts=2026-05-09T22:58:19.523993865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
ts=2026-05-09T22:58:36.101551839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49180"
ts=2026-05-09T22:58:42.269721134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:36104"
level=debug ts=2026-05-09T22:58:44.76243569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T22:58:49.527305635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
ts=2026-05-09T22:59:19.530287293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T22:59:27.520476416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T22:59:36.104982982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:59364"
ts=2026-05-09T22:59:45.563794807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34546"
ts=2026-05-09T22:59:49.532563545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T23:00:04.921336415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:00:06.108054802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:60864"
ts=2026-05-09T23:00:12.275302134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:53916"
ts=2026-05-09T23:00:19.535236247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
ts=2026-05-09T23:00:42.277863906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:44730"
ts=2026-05-09T23:00:49.537262851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T23:00:59.976749674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:01:12.281136394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:47904"
ts=2026-05-09T23:01:19.539350514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:01:42.283480274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:44798"
ts=2026-05-09T23:01:49.541317718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=info ts=2026-05-09T23:01:50.146348166Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-09T23:01:50.148556098Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T23:01:50.150228109Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T23:01:50.151731634Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T23:01:50.151756786Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T23:01:50.151769521Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-09T23:01:51.686646167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T23:01:51.688187048Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T23:02:06.118354329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35576"
ts=2026-05-09T23:02:12.286271377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:54504"
ts=2026-05-09T23:02:19.543681731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
level=debug ts=2026-05-09T23:02:45.647716653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:02:49.545336388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:03:15.585335075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45548"
ts=2026-05-09T23:03:19.547147334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:03:42.295260364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:43570"
level=debug ts=2026-05-09T23:03:42.980122334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:03:45.586878498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37070"
ts=2026-05-09T23:03:49.549323658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
ts=2026-05-09T23:04:19.552343402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T23:04:26.797522353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:04:49.555026247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
ts=2026-05-09T23:05:06.130212287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33560"
level=debug ts=2026-05-09T23:05:09.930624993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:05:19.55764392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:05:42.303741535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:59686"
ts=2026-05-09T23:05:45.597769657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59892"
ts=2026-05-09T23:05:49.560313088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T23:05:58.437973433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T23:05:58.439836629Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T23:06:12.306855138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:39342"
ts=2026-05-09T23:06:19.562819032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T23:06:44.819473229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:06:49.565150919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:07:06.140064852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54918"
ts=2026-05-09T23:07:15.605460877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53074"
ts=2026-05-09T23:07:19.567330677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T23:07:40.266915061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:07:49.570394281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
ts=2026-05-09T23:08:12.315800165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:35716"
ts=2026-05-09T23:08:19.573595391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"
level=debug ts=2026-05-09T23:08:29.617576295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:08:36.149322447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43866"
ts=2026-05-09T23:08:49.575430866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-980c1ace 10.129.0.76:7946"
ts=2026-05-09T23:09:19.577998157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wmnzh-2c6d6eb2 10.130.0.84:7946"
level=debug ts=2026-05-09T23:09:24.464366764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T23:09:49.581263895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g6wcd-08ec07d2 10.130.0.85:7946"