level=info ts=2026-04-20T13:14:07.648766979Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T13:14:07.648802208Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T13:14:07.649862713Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T13:14:07.650271558Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T13:14:07.650388786Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.83 inf=eth0
level=debug ts=2026-04-20T13:14:07.651857415Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T13:14:07.651964691Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.83 inf=eth0
level=info ts=2026-04-20T13:14:07.652047321Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-233e8617
ts=2026-04-20T13:14:07.652158704Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T13:14:07.652173409Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.83 advertisePort=7946
level=debug ts=2026-04-20T13:14:07.652192527Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.83 advertisePort=7946
level=info ts=2026-04-20T13:14:07.652312822Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T13:14:07.6574416Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T13:14:07.65746643Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T13:14:07.657481589Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.171609ms
level=error ts=2026-04-20T13:14:07.657492049Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T13:14:07.657505656Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T13:14:07.660223509Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T13:14:07.660243054Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T13:14:07.667536371Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-20T13:14:07.667536899Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T13:14:07.667582175Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T13:14:07.6675791Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T13:14:07.667601688Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-20T13:14:07.667604875Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T13:14:07.667737782Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T13:14:07.667749692Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-20T13:14:07.667897235Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T13:14:07.667908976Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T13:14:07.667965848Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T13:14:07.667963309Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T13:14:07.667981566Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T13:14:07.668159547Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T13:14:07.668183666Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T13:14:07.668234693Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T13:14:08.668809719Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T13:14:08.784036082Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T13:14:08.784116799Z caller=loki.go:599 msg="Loki started" startup_time=1.215309789s
ts=2026-04-20T13:14:08.790652946Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T13:14:08.790676568Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:14:11.531519953Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T13:14:11.531548334Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T13:14:13.784509369Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T13:14:13.784567291Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T13:14:19.019215685Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T13:14:19.019242659Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T13:14:23.423635946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:14:32.873343196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-04-20T13:14:32.876556526Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.68:7946: dial tcp 10.130.0.68:7946: connect: connection refused"
level=info ts=2026-04-20T13:14:32.876578287Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=25.219064461s
level=debug ts=2026-04-20T13:14:37.795905309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:14:38.070999691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:14:40.259231119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47770"
level=debug ts=2026-04-20T13:14:57.119512119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:15:08.074101092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:15:25.384612497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:15:38.076793778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:15:55.896872503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T13:15:55.898612699Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:16:08.07842972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:16:27.301951922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:16:38.08095405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:17:08.082683094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:17:20.038772798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35602"
level=debug ts=2026-04-20T13:17:34.543119413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:17:38.084541696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:18:08.086667739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:18:10.286119336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51298"
level=debug ts=2026-04-20T13:18:30.276102124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:18:38.089248882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:19:08.090690585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:19:20.049466611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44608"
level=debug ts=2026-04-20T13:19:28.208841185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T13:19:38.092620102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:20:08.095263479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:20:17.789621944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T13:20:17.791493253Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:20:33.66884484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:52286"
ts=2026-04-20T13:20:38.09892982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:20:40.304099863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41998"
ts=2026-04-20T13:20:50.053904398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:51378"
ts=2026-04-20T13:21:08.103194266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:21:13.006658145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:21:38.105602305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:21:50.056857076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36330"
level=debug ts=2026-04-20T13:22:06.32691997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:22:08.10753939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:22:20.05809077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36986"
ts=2026-04-20T13:22:33.675654667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:55210"
ts=2026-04-20T13:22:38.11056509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:22:40.318748789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49888"
ts=2026-04-20T13:22:50.058935628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:59990"
level=debug ts=2026-04-20T13:22:52.619005437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:23:08.112944664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:23:38.114617623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:23:47.616344794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:24:08.117542533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=info ts=2026-04-20T13:24:13.784873242Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T13:24:13.788830557Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:24:13.7910807Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:24:13.792930951Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:24:13.792961176Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T13:24:13.792974753Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-20T13:24:35.876967979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:24:35.878509731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:24:38.120409537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:25:08.122146804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:25:10.337678306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34778"
level=debug ts=2026-04-20T13:25:25.365378575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:25:38.123167878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:26:03.559199562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:26:08.124869901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:26:10.345834184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56248"
ts=2026-04-20T13:26:20.078067084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58164"
ts=2026-04-20T13:26:38.127280092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:26:40.349318713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46300"
ts=2026-04-20T13:26:50.079059547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40826"
level=debug ts=2026-04-20T13:27:03.02924336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:27:08.129438977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:27:10.353993783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53232"
ts=2026-04-20T13:27:20.08030933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33666"
ts=2026-04-20T13:27:33.698580002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:49476"
ts=2026-04-20T13:27:38.132243726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:27:39.574890515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:28:08.134071785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:28:33.702041959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:55550"
level=debug ts=2026-04-20T13:28:37.834296738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:28:37.835931313Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:28:38.135702706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:29:08.138222673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:29:14.050152505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:29:20.089316673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48034"
ts=2026-04-20T13:29:33.705114628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:53862"
ts=2026-04-20T13:29:38.142000797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:29:40.372115997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41098"
ts=2026-04-20T13:29:50.090365798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40384"
level=debug ts=2026-04-20T13:30:02.321835224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:30:08.144045823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:30:38.146651462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:30:40.378674819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45338"
level=debug ts=2026-04-20T13:30:57.279785532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:31:03.711523065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:43594"
ts=2026-04-20T13:31:08.148606982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:31:20.094976393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60796"
ts=2026-04-20T13:31:33.712817403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:47830"
ts=2026-04-20T13:31:38.151164665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:31:40.386146701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34736"
level=debug ts=2026-04-20T13:31:54.559614045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:32:08.154277474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:32:33.717756471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:51734"
ts=2026-04-20T13:32:38.156014559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:32:50.103748785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:53518"
level=debug ts=2026-04-20T13:32:51.31736301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:32:51.319152952Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:33:08.158657784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:33:38.161545858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:33:47.841199396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:34:03.725066516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38162"
ts=2026-04-20T13:34:08.162978338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=error ts=2026-04-20T13:34:13.797378874Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:34:13.79915184Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:34:13.800818744Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:34:13.800843363Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:34:20.110492218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36114"
level=debug ts=2026-04-20T13:34:33.735953723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:34:38.16402929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:35:08.165830562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:35:35.494206877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:35:38.167550283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:35:50.114245601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33208"
ts=2026-04-20T13:36:08.168697488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:36:10.420985229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42306"
level=debug ts=2026-04-20T13:36:25.343140218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:36:33.734035668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:53266"
ts=2026-04-20T13:36:38.171447143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:36:40.424819063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58870"
ts=2026-04-20T13:37:03.735838232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:42010"
ts=2026-04-20T13:37:08.174626383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:37:24.934042979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:37:24.936162496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:37:38.177263526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:37:40.431379094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46190"
ts=2026-04-20T13:38:08.180053217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:38:24.757592213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:38:33.741072562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60804"
ts=2026-04-20T13:38:38.182643922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:38:40.43881244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56250"
ts=2026-04-20T13:38:50.128538087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:47210"
level=debug ts=2026-04-20T13:39:04.394643216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:39:08.186637749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:39:38.190341594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:39:40.445677719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48378"
level=debug ts=2026-04-20T13:39:43.64620467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:40:08.194017833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:40:38.196321616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:40:40.453751419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34338"
level=debug ts=2026-04-20T13:40:46.034881368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:41:08.197849157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:41:20.945046638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:41:20.947136461Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:41:38.199430064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:41:40.460628287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51254"
ts=2026-04-20T13:42:08.201165175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:42:10.463436663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33822"
level=debug ts=2026-04-20T13:42:15.449381107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:42:38.202461267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:42:53.933752403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:43:08.205259739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:43:28.928847892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:43:38.208085143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:43:40.47507295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42638"
ts=2026-04-20T13:43:50.151039838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34534"
ts=2026-04-20T13:44:03.769350848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38196"
ts=2026-04-20T13:44:08.211130688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=error ts=2026-04-20T13:44:13.797231144Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:44:13.799139262Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:44:13.80090988Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:44:13.800940224Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T13:44:16.973191244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:44:33.770933528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:45420"
ts=2026-04-20T13:44:38.213770783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:44:40.482363542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43346"
ts=2026-04-20T13:44:50.153664848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44796"
ts=2026-04-20T13:45:08.216589963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:45:19.004050567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:45:19.006331729Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:45:20.155500179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35924"
ts=2026-04-20T13:45:33.774189822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:47854"
ts=2026-04-20T13:45:38.219018629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:46:08.220528191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:46:20.02149566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:46:38.222146584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:46:50.161595644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60332"
level=debug ts=2026-04-20T13:46:56.576613033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:47:08.223799061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:47:38.225150103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:47:42.54815118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:47:50.164640655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:39466"
ts=2026-04-20T13:48:08.228167909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:48:20.165847995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38676"
ts=2026-04-20T13:48:38.230409748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:48:42.511262002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:48:50.167025162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57278"
ts=2026-04-20T13:49:03.793406774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46004"
ts=2026-04-20T13:49:08.232280104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:49:38.234271873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:49:40.517187992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33124"
level=debug ts=2026-04-20T13:49:42.223572331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:49:42.225285815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:50:03.795223097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:58944"
ts=2026-04-20T13:50:08.235455967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:50:10.520380618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60130"
level=debug ts=2026-04-20T13:50:34.822991002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:50:38.238071567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:51:03.800870418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:40786"
ts=2026-04-20T13:51:08.240375689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:51:31.56978751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:51:38.242072975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:52:08.244884033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:52:10.535161965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47640"
level=debug ts=2026-04-20T13:52:16.310235884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:52:20.183084628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42818"
ts=2026-04-20T13:52:38.247686278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:52:50.184586753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48290"
level=debug ts=2026-04-20T13:53:06.286552138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:53:08.25015828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:53:10.541929997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48920"
ts=2026-04-20T13:53:33.813107031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35634"
ts=2026-04-20T13:53:38.25397882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:54:06.579078085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:54:06.580774376Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:54:08.256823953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:54:10.547651879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47986"
level=error ts=2026-04-20T13:54:13.795737484Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:54:13.797480636Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:54:13.799259125Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T13:54:13.799287267Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:54:33.816978938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:41602"
ts=2026-04-20T13:54:38.258761151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:54:50.192124045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36822"
level=debug ts=2026-04-20T13:55:02.681271872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:55:03.818697765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:57692"
ts=2026-04-20T13:55:08.259769772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:55:20.193993131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40734"
ts=2026-04-20T13:55:38.261244703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
level=debug ts=2026-04-20T13:55:39.306009941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:56:03.820997399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46246"
ts=2026-04-20T13:56:08.262324878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:56:15.925754397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:56:33.82278243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:33594"
ts=2026-04-20T13:56:38.264434573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T13:56:40.567305657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46668"
ts=2026-04-20T13:57:03.824237291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:41468"
ts=2026-04-20T13:57:08.267161781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:57:10.570328951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49532"
level=debug ts=2026-04-20T13:57:15.828253589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:57:38.268256833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:57:50.206196759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:54164"
ts=2026-04-20T13:58:03.828634043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:39486"
ts=2026-04-20T13:58:08.269994132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
level=debug ts=2026-04-20T13:58:11.565380137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T13:58:11.567067251Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T13:58:33.829523534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38232"
ts=2026-04-20T13:58:38.272406621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T13:58:46.528942604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:59:08.274371638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574d0e21 10.129.0.84:7946"
ts=2026-04-20T13:59:10.58586681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39792"
ts=2026-04-20T13:59:20.215168489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46600"
level=debug ts=2026-04-20T13:59:20.674048619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:59:33.831395323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54100"
ts=2026-04-20T13:59:38.276108565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T13:59:50.21654405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42126"
level=debug ts=2026-04-20T14:00:02.938817241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:00:08.277234232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T14:00:20.218096685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44466"
ts=2026-04-20T14:00:38.280285396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
level=debug ts=2026-04-20T14:00:55.902072025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:01:03.83793891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:57682"
ts=2026-04-20T14:01:08.282986562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"
ts=2026-04-20T14:01:10.599591466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48088"
level=debug ts=2026-04-20T14:01:35.524797313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:01:35.526680944Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:01:38.28582467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r7fmz-ebb5357c 10.130.0.67:7946"
ts=2026-04-20T14:02:08.288277218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8258m-18f7b751 10.129.0.79:7946"