level=info ts=2026-04-21T03:58:44.65683504Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T03:58:44.656882042Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T03:58:44.65931977Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T03:58:44.659734558Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T03:58:44.65984342Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0
level=debug ts=2026-04-21T03:58:44.66038248Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T03:58:44.660470912Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0
level=info ts=2026-04-21T03:58:44.661748106Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-0f8eaf53
ts=2026-04-21T03:58:44.661828605Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T03:58:44.661845904Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946
level=debug ts=2026-04-21T03:58:44.661864907Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946
level=info ts=2026-04-21T03:58:44.662043397Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T03:58:44.670464917Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T03:58:44.670499601Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T03:58:44.670513108Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.472289ms
level=error ts=2026-04-21T03:58:44.670522773Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T03:58:44.670539469Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T03:58:44.676004808Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T03:58:44.676028742Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T03:58:44.681800596Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T03:58:44.682136671Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-21T03:58:44.682152609Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T03:58:44.68225745Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T03:58:44.682286735Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T03:58:44.682295137Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T03:58:44.682310568Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T03:58:44.682315524Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-21T03:58:44.682319814Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T03:58:44.682447488Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T03:58:44.682494874Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T03:58:44.68253637Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T03:58:44.68255663Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T03:58:44.682964136Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T03:58:44.818395236Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T03:58:44.818475124Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T03:58:45.819335908Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T03:58:45.925879553Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T03:58:45.925961562Z caller=loki.go:599 msg="Loki started" startup_time=1.35030509s
ts=2026-04-21T03:58:46.121804951Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T03:58:46.121830325Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T03:58:49.055679662Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T03:58:49.055703247Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T03:58:50.926767911Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T03:58:50.926822542Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T03:58:53.833247937Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T03:58:53.833273187Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T03:59:02.555440323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T03:59:04.985170663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43670"
ts=2026-04-21T03:59:06.406282721Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.81:7946: dial tcp 10.129.0.81:7946: connect: connection refused"
ts=2026-04-21T03:59:06.40644169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
ts=2026-04-21T03:59:06.407579895Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.88:7946: dial tcp 10.129.0.88:7946: connect: connection refused"
ts=2026-04-21T03:59:06.408257982Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.85:7946: dial tcp 10.129.0.85:7946: connect: connection refused"
ts=2026-04-21T03:59:06.408346224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.89:7946"
ts=2026-04-21T03:59:06.408360899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36002"
ts=2026-04-21T03:59:06.408800177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.87:7946"
level=info ts=2026-04-21T03:59:06.409079964Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.738534388s
ts=2026-04-21T03:59:07.105698779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:34642"
ts=2026-04-21T03:59:07.811264232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44660"
level=debug ts=2026-04-21T03:59:23.537702768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T03:59:31.921773633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:54226"
ts=2026-04-21T03:59:33.025238342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T03:59:39.700632704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T03:59:59.976219132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:00:01.923315411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:60508"
ts=2026-04-21T04:00:03.02626062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:00:25.982540005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38138"
ts=2026-04-21T04:00:33.027392123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:00:38.743443582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:00:38.745170449Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:01:03.028657535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:01:24.77465866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:01:33.030104202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:01:55.987169162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54458"
ts=2026-04-21T04:02:03.031674283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:02:12.477431053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:02:31.92936384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33306"
ts=2026-04-21T04:02:33.032887943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:02:55.990136337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40450"
ts=2026-04-21T04:03:03.034353912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:03:12.255289369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:03:18.431106752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:58866"
ts=2026-04-21T04:03:33.036160979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:04:03.037284254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:04:13.782509864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:04:31.933902813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:34406"
ts=2026-04-21T04:04:33.038497806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:05:03.039931493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:05:25.997461461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51524"
level=debug ts=2026-04-21T04:05:30.550679434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:05:30.552417438Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:05:31.935842654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:50956"
ts=2026-04-21T04:05:33.041474189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:05:48.436532265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39768"
ts=2026-04-21T04:06:03.043195679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:06:18.161981554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:06:33.044878203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:06:48.439925057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44658"
level=debug ts=2026-04-21T04:06:53.825859436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:07:03.046021345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:07:26.002978185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50366"
ts=2026-04-21T04:07:33.047345859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:07:46.063144088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:08:03.049054978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:08:25.251590611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:08:33.050063409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=info ts=2026-04-21T04:08:50.926992236Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T04:08:50.934004493Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:08:50.935970417Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:08:50.937763344Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:08:50.937789603Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T04:08:50.937801354Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T04:09:03.051823216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:09:09.902288383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:09:09.903938161Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:09:26.009108977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35166"
ts=2026-04-21T04:09:33.05319249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:09:57.066415658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:10:03.054366882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:10:18.449347601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:53022"
ts=2026-04-21T04:10:31.948218203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:45194"
ts=2026-04-21T04:10:33.056077008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:10:44.27447113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:10:48.451155159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:42134"
ts=2026-04-21T04:11:01.949264898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:39102"
ts=2026-04-21T04:11:03.056968651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:11:26.124622048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:11:31.95105285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:40130"
ts=2026-04-21T04:11:33.058745581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:11:48.454099468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45576"
ts=2026-04-21T04:11:56.016646927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42088"
ts=2026-04-21T04:12:03.060260452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:12:19.702232218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:12:26.017931125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59380"
ts=2026-04-21T04:12:33.0620924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:12:56.019747799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53748"
ts=2026-04-21T04:13:01.955182512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33932"
ts=2026-04-21T04:13:03.063761709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:13:08.756062555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:13:08.758170924Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:13:26.021417126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56804"
ts=2026-04-21T04:13:33.065260988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:13:44.041864377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:14:03.067059688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:14:18.460965252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:33126"
ts=2026-04-21T04:14:26.024739218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44446"
ts=2026-04-21T04:14:31.959273809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:39508"
ts=2026-04-21T04:14:33.068101859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:14:34.893798105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:15:03.069023571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:15:32.455708793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:15:33.070323973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:15:56.02887843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47614"
ts=2026-04-21T04:16:01.964273634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:50588"
ts=2026-04-21T04:16:03.071357634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:16:25.799109028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:16:26.030669051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50012"
ts=2026-04-21T04:16:33.07244406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:16:56.031950224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50446"
ts=2026-04-21T04:17:03.074275551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:17:09.380618452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:17:09.382217361Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:17:31.968507756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:60182"
ts=2026-04-21T04:17:33.075266374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:17:56.034832737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41090"
ts=2026-04-21T04:18:01.969516604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:59978"
ts=2026-04-21T04:18:03.077108521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:18:03.75245207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:18:33.078441359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:18:48.472988671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:47442"
level=debug ts=2026-04-21T04:18:50.804195061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:18:50.941417052Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:18:50.943595238Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:18:50.945613443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:18:50.945640895Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:19:03.079708898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:19:18.474506498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45684"
ts=2026-04-21T04:19:26.039488578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40398"
ts=2026-04-21T04:19:33.081370769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:19:37.124292652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:19:56.040582909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60188"
ts=2026-04-21T04:20:01.976035816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:41400"
ts=2026-04-21T04:20:03.082377412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:20:11.737728249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:20:31.977696886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:58634"
ts=2026-04-21T04:20:33.083559326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:20:56.043477947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44944"
ts=2026-04-21T04:21:01.979516347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:48382"
ts=2026-04-21T04:21:03.085281212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:21:06.586486976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:21:06.588528875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:21:31.981022121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36406"
ts=2026-04-21T04:21:33.086923139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:21:42.435391657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:22:03.088453948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:22:17.744940245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:22:33.090061977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:23:03.091693285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:23:03.168284671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:23:26.051014786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51076"
ts=2026-04-21T04:23:31.98708519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:39956"
ts=2026-04-21T04:23:33.092810704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:23:44.644241843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:23:48.486227405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:43796"
ts=2026-04-21T04:23:56.05197374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56592"
ts=2026-04-21T04:24:03.09410427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:24:23.050534796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:24:23.052271459Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:24:31.990136109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:49656"
ts=2026-04-21T04:24:33.095770225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:24:48.488984297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:57740"
ts=2026-04-21T04:24:56.055460803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43592"
ts=2026-04-21T04:25:03.097159877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:25:18.490555617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36094"
level=debug ts=2026-04-21T04:25:24.10950095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:25:31.993343001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:44338"
ts=2026-04-21T04:25:33.098993177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:25:59.433429756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:26:01.994750745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:40320"
ts=2026-04-21T04:26:03.100195337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:26:18.493894877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:53894"
ts=2026-04-21T04:26:26.060367221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51838"
ts=2026-04-21T04:26:33.101894763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:26:41.530725496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:26:48.49575051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:35468"
ts=2026-04-21T04:27:03.103700833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:27:33.105445982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:27:36.992131951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:28:01.999296133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:44162"
ts=2026-04-21T04:28:03.106954658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:28:18.500927278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:57260"
ts=2026-04-21T04:28:26.066315665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40098"
level=debug ts=2026-04-21T04:28:30.223433354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:28:30.225257598Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:28:33.108020825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:28:48.501687979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:54784"
level=error ts=2026-04-21T04:28:50.942006133Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:28:50.943863308Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:28:50.945423206Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:28:50.945451425Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:29:03.109294121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:29:04.68999583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:29:33.111037355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:29:52.111102013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:30:02.004127483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:58534"
ts=2026-04-21T04:30:03.112368943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:30:26.071752711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55542"
ts=2026-04-21T04:30:32.00560127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33770"
ts=2026-04-21T04:30:33.114146247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:30:53.905956625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:30:56.073344906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55970"
ts=2026-04-21T04:31:03.115390615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:31:18.509282392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:37634"
ts=2026-04-21T04:31:32.008839738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:53416"
ts=2026-04-21T04:31:33.116665635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:31:45.88294575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:32:02.010462763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:35004"
ts=2026-04-21T04:32:03.118303526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:32:21.51929369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:32:21.520951318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:32:33.12020865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:32:48.513364117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:37460"
ts=2026-04-21T04:33:03.12094042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:33:03.511844317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:33:32.014533006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:54218"
ts=2026-04-21T04:33:33.122088364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:33:48.516953956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:51202"
level=debug ts=2026-04-21T04:33:55.222616945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:33:56.083022425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55782"
ts=2026-04-21T04:34:02.015702418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:46314"
ts=2026-04-21T04:34:03.123351559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:34:18.518205118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:58808"
ts=2026-04-21T04:34:33.125106354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:34:36.231438736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:35:03.126515832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:35:26.087579267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39960"
level=debug ts=2026-04-21T04:35:26.691706634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:35:33.127333282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:36:02.567860755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:36:02.569600955Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:36:03.128481381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:36:26.090684709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46698"
ts=2026-04-21T04:36:33.129841325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:36:48.525809086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:59200"
ts=2026-04-21T04:36:56.091518429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46220"
level=debug ts=2026-04-21T04:36:57.066455091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:37:03.131094112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:37:33.132607951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:37:56.093730581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53232"
level=debug ts=2026-04-21T04:37:57.988040075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:38:03.133678986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:38:33.134745094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:38:35.376712034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:38:50.941789895Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:38:50.943903305Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:38:50.945789856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:38:50.945818307Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:38:56.096798321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46246"
ts=2026-04-21T04:39:03.135531172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:39:30.54928099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:39:32.030372712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:37836"
ts=2026-04-21T04:39:33.137298907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:40:03.138981047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:40:13.231058699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:40:13.233140736Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:40:18.534943413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45856"
ts=2026-04-21T04:40:32.032423484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:35394"
ts=2026-04-21T04:40:33.140616286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:40:48.535878072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:50142"
level=debug ts=2026-04-21T04:41:01.219071123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:41:02.03394856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:58228"
ts=2026-04-21T04:41:03.14235403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:41:32.035787759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:52072"
ts=2026-04-21T04:41:33.144084798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:41:58.756341585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:42:03.145466075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:42:26.106178623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33646"
ts=2026-04-21T04:42:33.146659762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:42:43.44113303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:43:03.148480968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:43:28.680950078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:43:32.040608658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:44264"
ts=2026-04-21T04:43:33.149397059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:44:02.04216917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:40078"
ts=2026-04-21T04:44:03.151142394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:44:07.7568239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:44:07.758512014Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:44:18.545856797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39908"
ts=2026-04-21T04:44:26.112498314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60256"
ts=2026-04-21T04:44:33.152778703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:45:02.044942643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:49580"
ts=2026-04-21T04:45:03.15420628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:45:09.111125567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:45:18.54887245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:34812"
ts=2026-04-21T04:45:33.155222656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:45:44.093925539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:45:56.116533252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41950"
ts=2026-04-21T04:46:03.156868485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:46:23.525098719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:46:32.048848114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:37590"
ts=2026-04-21T04:46:33.158186727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:46:56.119122285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46004"
ts=2026-04-21T04:47:03.158951099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:47:14.166111885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:47:18.554128508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:40374"
ts=2026-04-21T04:47:33.160294059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:47:48.555868573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:34810"
ts=2026-04-21T04:48:03.162045422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:48:16.485581002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:48:16.487737056Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:48:18.556833896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:42880"
ts=2026-04-21T04:48:33.16372817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=error ts=2026-04-21T04:48:50.942554993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:48:50.944501644Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:48:50.946316542Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:48:50.946341682Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:49:03.165218466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:49:12.124238849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:49:33.166451153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:49:48.560659031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:51482"
ts=2026-04-21T04:49:56.127062687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51002"
ts=2026-04-21T04:50:02.057710714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33148"
ts=2026-04-21T04:50:03.168126958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
level=debug ts=2026-04-21T04:50:10.590006287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:50:33.168884485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:50:48.564136248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36856"
ts=2026-04-21T04:51:02.059617832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:34494"
ts=2026-04-21T04:51:03.170145308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:51:05.225279581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:51:26.130861217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48298"
ts=2026-04-21T04:51:33.171903768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:51:48.566937933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:48028"
level=debug ts=2026-04-21T04:51:50.498107086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:52:02.062986022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36048"
ts=2026-04-21T04:52:03.172926381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-73c7e680 10.129.0.90:7946"
ts=2026-04-21T04:52:33.174534757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:52:34.669675321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:52:34.671367352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:52:48.568978761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:43778"
ts=2026-04-21T04:53:03.176279144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
level=debug ts=2026-04-21T04:53:13.026760359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:53:33.177903227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:53:56.136600998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42362"
ts=2026-04-21T04:54:03.178776587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
level=debug ts=2026-04-21T04:54:06.845720049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:54:33.180550831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-jgx85-e9c4773d 10.129.0.83:7946"
ts=2026-04-21T04:54:48.574342307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:56238"
level=debug ts=2026-04-21T04:54:50.395759045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:55:03.182424996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gqw5j-5ba9affd 10.129.0.87:7946"
ts=2026-04-21T04:55:18.575607082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52476"