level=info ts=2026-04-22T10:06:48.616008409Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T10:06:48.61603598Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T10:06:48.616943561Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T10:06:48.617249642Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T10:06:48.617325067Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
level=debug ts=2026-04-22T10:06:48.618295961Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T10:06:48.618364009Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
level=info ts=2026-04-22T10:06:48.618423341Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-0c1fad4c
ts=2026-04-22T10:06:48.618520979Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T10:06:48.618549672Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=debug ts=2026-04-22T10:06:48.618569452Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=info ts=2026-04-22T10:06:48.618707474Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T10:06:48.624965459Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T10:06:48.624983413Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T10:06:48.624995731Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.290424ms
level=error ts=2026-04-22T10:06:48.625004815Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T10:06:48.625018663Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T10:06:48.627773321Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T10:06:48.627787687Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T10:06:48.632980939Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T10:06:48.633002419Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T10:06:48.63300872Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T10:06:48.633018724Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T10:06:48.633050234Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T10:06:48.633073282Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T10:06:48.633091351Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T10:06:48.633311766Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-22T10:06:48.633321885Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-22T10:06:48.633327226Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T10:06:48.633385587Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T10:06:48.633411866Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T10:06:48.633413124Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T10:06:48.633551161Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T10:06:48.74551401Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T10:06:48.74556591Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T10:06:49.746540454Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T10:06:49.916004959Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T10:06:49.91605229Z caller=loki.go:599 msg="Loki started" startup_time=1.358909695s
ts=2026-04-22T10:06:50.337804928Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T10:06:50.337825502Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:06:53.639462643Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T10:06:53.639484842Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T10:06:54.916774281Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T10:06:54.916821057Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T10:06:58.709749119Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T10:06:58.709768451Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T10:07:05.974413764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:07:08.710833262Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
ts=2026-04-22T10:07:08.711862671Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.77:7946: dial tcp 10.128.0.77:7946: connect: connection refused"
ts=2026-04-22T10:07:08.71325587Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.79:7946: dial tcp 10.128.0.79:7946: connect: connection refused"
ts=2026-04-22T10:07:08.713344583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.64:7946"
ts=2026-04-22T10:07:08.713361019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44564"
level=info ts=2026-04-22T10:07:08.71356904Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.088544669s
ts=2026-04-22T10:07:11.590369819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58622"
ts=2026-04-22T10:07:12.546362152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51344"
ts=2026-04-22T10:07:15.901559598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39920"
level=debug ts=2026-04-22T10:07:25.499760372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:07:27.715282271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:07:29.922462621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52630"
ts=2026-04-22T10:07:30.198419443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53154"
ts=2026-04-22T10:07:41.545720683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42666"
level=debug ts=2026-04-22T10:07:47.7521294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:07:57.717353744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
level=debug ts=2026-04-22T10:08:08.550401078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:08:27.720039485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:08:30.204666581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43990"
ts=2026-04-22T10:08:41.548052622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43980"
level=debug ts=2026-04-22T10:08:47.218630477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T10:08:47.220445605Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T10:08:57.721509707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:09:11.549173028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35960"
ts=2026-04-22T10:09:27.722446104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
level=debug ts=2026-04-22T10:09:29.850622116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:09:57.724662067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:10:27.726474872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
level=debug ts=2026-04-22T10:10:41.460009797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:10:57.72862461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:11:27.730684329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:11:30.222474859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38954"
ts=2026-04-22T10:11:57.731396112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:11:59.940515307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48022"
ts=2026-04-22T10:12:00.225049705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49112"
level=debug ts=2026-04-22T10:12:01.448261688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:12:11.560459848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33234"
ts=2026-04-22T10:12:27.733973827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:12:41.561830342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40898"
ts=2026-04-22T10:12:57.736300009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:12:59.943075361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55204"
ts=2026-04-22T10:13:00.23146315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55150"
level=debug ts=2026-04-22T10:13:01.145829561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T10:13:27.737549914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:13:29.944314667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55714"
level=debug ts=2026-04-22T10:13:55.014114432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T10:13:55.015616055Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T10:13:57.739258513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:13:59.945682696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58522"
ts=2026-04-22T10:14:27.740700368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
ts=2026-04-22T10:14:29.947205714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59304"
level=debug ts=2026-04-22T10:14:51.030769476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:14:57.742952297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:14:59.948030655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46544"
ts=2026-04-22T10:15:00.243519449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40420"
ts=2026-04-22T10:15:27.744695472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07681ff8 10.130.0.65:7946"
ts=2026-04-22T10:15:29.949476855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40132"
ts=2026-04-22T10:15:30.246081297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41090"
level=debug ts=2026-04-22T10:15:35.542415115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:15:57.74697094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:16:00.249312611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55320"
ts=2026-04-22T10:16:11.576550372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58948"
ts=2026-04-22T10:16:27.748673931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
ts=2026-04-22T10:16:29.953546522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38356"
level=debug ts=2026-04-22T10:16:32.598453666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T10:16:54.916973879Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T10:16:54.919956785Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T10:16:54.921672361Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T10:16:54.923112397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T10:16:54.923136714Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T10:16:54.923147899Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T10:16:57.750467452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:16:59.954267767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38624"
ts=2026-04-22T10:17:27.752259368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
level=debug ts=2026-04-22T10:17:27.982467555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:17:57.754568063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:18:00.260475884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56980"
level=debug ts=2026-04-22T10:18:10.345958549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T10:18:10.347560894Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T10:18:27.757548685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
ts=2026-04-22T10:18:29.959957757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44394"
level=debug ts=2026-04-22T10:18:55.041412612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:18:57.759198989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
ts=2026-04-22T10:18:59.961253575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44410"
ts=2026-04-22T10:19:27.760833798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sr44s-6c5152c8 10.129.0.78:7946"
level=debug ts=2026-04-22T10:19:43.141745417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:19:57.76307839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
level=debug ts=2026-04-22T10:20:25.02106994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T10:20:27.764677318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
ts=2026-04-22T10:20:57.765707068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d44q8-ee432abe 10.130.0.61:7946"
ts=2026-04-22T10:21:00.278286822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37962"