level=info ts=2026-04-16T16:05:54.855961124Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T16:05:54.855995588Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-16T16:05:54.856777713Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T16:05:54.857066804Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T16:05:54.85713172Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.104 inf=eth0
level=debug ts=2026-04-16T16:05:54.857989729Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T16:05:54.858052629Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.104 inf=eth0
level=info ts=2026-04-16T16:05:54.858126414Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-a04a5887
ts=2026-04-16T16:05:54.858192922Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T16:05:54.85820565Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.104 advertisePort=7946
level=debug ts=2026-04-16T16:05:54.858250155Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.104 advertisePort=7946
level=info ts=2026-04-16T16:05:54.858362357Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-16T16:05:54.864868281Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T16:05:54.864887089Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T16:05:54.864896124Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.537444ms
level=error ts=2026-04-16T16:05:54.864902759Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-16T16:05:54.864913392Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-16T16:05:54.867299123Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T16:05:54.867315114Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T16:05:54.876172076Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-16T16:05:54.876173539Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-16T16:05:54.87619653Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-16T16:05:54.876200156Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-16T16:05:54.876206383Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-16T16:05:54.876251553Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-16T16:05:54.876276622Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T16:05:54.876308987Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-16T16:05:54.876414327Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-16T16:05:54.876425315Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-16T16:05:54.876472741Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-16T16:05:54.876488679Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-16T16:05:54.876511272Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T16:05:54.876667304Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-16T16:05:55.057059318Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=info ts=2026-04-16T16:05:55.057115664Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-16T16:05:55.237483745Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-16T16:05:55.23751663Z caller=loki.go:599 msg="Loki started" startup_time=443.45047ms
ts=2026-04-16T16:05:56.709514488Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T16:05:56.709541818Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:05:59.888319506Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T16:05:59.88834296Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T16:06:00.237620824Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-16T16:06:00.237656049Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-16T16:06:06.914333144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-04-16T16:06:06.916780556Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.66:7946: dial tcp 10.129.0.66:7946: connect: connection refused"
level=info ts=2026-04-16T16:06:06.916797199Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=12.051879756s
level=debug ts=2026-04-16T16:06:08.902503492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:06:19.263366067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:33990"
level=debug ts=2026-04-16T16:06:24.671452814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:06:26.959739842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:38176"
ts=2026-04-16T16:06:28.634460783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:06:30.371849854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:51938"
level=debug ts=2026-04-16T16:06:40.426010993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:06:58.636017397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:07:07.694340482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:07:28.637638783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:07:30.377564847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57024"
ts=2026-04-16T16:07:34.378482241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51662"
level=debug ts=2026-04-16T16:07:42.385639121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T16:07:42.387213486Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:07:56.966765209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:52130"
ts=2026-04-16T16:07:58.640423877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
level=debug ts=2026-04-16T16:08:26.774580454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:08:28.64273205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:08:30.383104334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:44148"
ts=2026-04-16T16:08:58.645004037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:09:04.386862649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37796"
ts=2026-04-16T16:09:28.646697602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:09:30.388864553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:42450"
level=debug ts=2026-04-16T16:09:33.089089719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:09:58.64932381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:10:04.39267972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32946"
ts=2026-04-16T16:10:28.651854293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:10:45.023791168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:10:56.982260425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:45712"
ts=2026-04-16T16:10:58.653536455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:11:00.396774672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57198"
ts=2026-04-16T16:11:04.398192982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46724"
ts=2026-04-16T16:11:28.655796187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:11:45.642884468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:11:56.986438187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39982"
ts=2026-04-16T16:11:58.657512654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:12:00.401855584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:60982"
ts=2026-04-16T16:12:28.659107338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:12:30.404470943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54806"
level=debug ts=2026-04-16T16:12:58.323084798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T16:12:58.3246237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:12:58.660732643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:13:00.407627757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38746"
ts=2026-04-16T16:13:26.99352723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:51700"
ts=2026-04-16T16:13:28.663439619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
level=debug ts=2026-04-16T16:13:38.258493584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:13:58.665195976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:14:04.413409104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39942"
ts=2026-04-16T16:14:28.667309852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:14:30.57059289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:14:34.416464631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59780"
ts=2026-04-16T16:14:58.670948609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:15:00.41867933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:47414"
level=debug ts=2026-04-16T16:15:10.313005709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:15:28.674457629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:15:34.42302897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46410"
level=debug ts=2026-04-16T16:15:56.435002227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:15:58.677353796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=info ts=2026-04-16T16:16:00.237895063Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-16T16:16:00.24061075Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:16:00.242652111Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:16:00.244541078Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:16:00.244571056Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T16:16:00.244584185Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-16T16:16:27.009310513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:54070"
ts=2026-04-16T16:16:28.679516028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:16:30.427192443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:51534"
ts=2026-04-16T16:16:34.428302006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52524"
level=debug ts=2026-04-16T16:16:50.132658634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:16:50.13440668Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:16:57.010516645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42924"
ts=2026-04-16T16:16:58.68156737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:17:28.683794366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:17:46.878568404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:17:58.685749941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:18:25.16834087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:18:27.017606104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41812"
ts=2026-04-16T16:18:28.687718074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:18:30.440274874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:37392"
ts=2026-04-16T16:18:58.690443723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
level=debug ts=2026-04-16T16:19:17.687713064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:19:28.695110198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:19:57.025538657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:34056"
ts=2026-04-16T16:19:58.697876672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:20:00.44851633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49014"
level=debug ts=2026-04-16T16:20:08.01130118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:20:28.700928734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:20:30.451310084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:35808"
level=debug ts=2026-04-16T16:20:48.246067907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:20:48.248667744Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:20:57.030625826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:51690"
ts=2026-04-16T16:20:58.704385268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:21:28.712608107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:21:49.330512656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:21:58.715690448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:22:00.457353128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:36118"
ts=2026-04-16T16:22:27.038897007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:37202"
ts=2026-04-16T16:22:28.718459288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:22:30.459824647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:40238"
level=debug ts=2026-04-16T16:22:44.442387865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:22:58.721819634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:23:00.462937799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:36600"
ts=2026-04-16T16:23:04.462421202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54548"
level=debug ts=2026-04-16T16:23:25.629323096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:23:28.72540672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74978b71 10.130.0.74:7946"
ts=2026-04-16T16:23:58.727867778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:24:00.468177382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:56694"
level=debug ts=2026-04-16T16:24:21.843865193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:24:28.73006617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:24:34.470546264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41762"
ts=2026-04-16T16:24:57.053349605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:34810"
ts=2026-04-16T16:24:58.732528126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:25:04.473472946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48526"
level=debug ts=2026-04-16T16:25:08.892725448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:25:08.894915379Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:25:27.055154314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:59136"
ts=2026-04-16T16:25:28.734524014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:25:30.498520442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:53644"
level=debug ts=2026-04-16T16:25:54.665885694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:25:58.737091445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=error ts=2026-04-16T16:26:00.248479594Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:26:00.251265669Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:26:00.253700472Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:26:00.253731985Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:26:27.06043505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42764"
ts=2026-04-16T16:26:28.740400323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:26:30.503166243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49098"
ts=2026-04-16T16:26:34.769441822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49104"
ts=2026-04-16T16:26:34.782617432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49114"
level=debug ts=2026-04-16T16:26:38.940534665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:26:58.743069362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-38fc950b 10.130.0.74:7946"
ts=2026-04-16T16:27:00.463327134Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-74978b71 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T16:27:01.859104096Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-74978b71 (timeout reached)"
ts=2026-04-16T16:27:04.488002128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32988"
ts=2026-04-16T16:27:04.859400807Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-74978b71 has failed, no acks received"
ts=2026-04-16T16:27:16.4490601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:37830"
ts=2026-04-16T16:27:27.063775495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:50374"
ts=2026-04-16T16:27:28.745552913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:27:39.332036927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:27:58.747109341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:28:26.079109784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:28:27.067950742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41660"
ts=2026-04-16T16:28:28.74948555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:28:46.458281979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57948"
ts=2026-04-16T16:28:58.752100036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:29:23.753319773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:29:23.755664932Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:29:28.754740935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-38fc950b 10.130.0.74:7946"
ts=2026-04-16T16:29:58.758006493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:30:23.926497316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:30:27.081905047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:45376"
ts=2026-04-16T16:30:28.76096956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:30:46.468324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:44764"
ts=2026-04-16T16:30:57.083486915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46446"
ts=2026-04-16T16:30:58.764141296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:31:11.287690187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:31:16.470530701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:40728"
ts=2026-04-16T16:31:27.085242858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46456"
ts=2026-04-16T16:31:28.767239402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-38fc950b 10.130.0.74:7946"
ts=2026-04-16T16:31:58.769792044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:32:07.176036254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:32:28.772478458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-38fc950b 10.130.0.74:7946"
ts=2026-04-16T16:32:34.532672208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34380"
ts=2026-04-16T16:32:46.478319812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:40788"
level=debug ts=2026-04-16T16:32:49.55095129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:32:58.774955653Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-38fc950b failed: dial tcp 10.130.0.74:7946: connect: connection refused"
ts=2026-04-16T16:33:11.859765193Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-38fc950b (timeout reached)"
ts=2026-04-16T16:33:12.537358964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:35910"
ts=2026-04-16T16:33:12.548547473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:35914"
ts=2026-04-16T16:33:14.859652024Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-38fc950b has failed, no acks received"
ts=2026-04-16T16:33:26.859087842Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-38fc950b (timeout reached)"
ts=2026-04-16T16:33:27.09566784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:47228"
ts=2026-04-16T16:33:28.775803708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:33:29.858981003Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-38fc950b has failed, no acks received"
ts=2026-04-16T16:33:29.86430931Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-38fc950b as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-16T16:33:46.409385566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:33:46.411668568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:33:57.096623787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:33408"
ts=2026-04-16T16:33:58.777466198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:34:26.64673563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:34:27.097627779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:59688"
ts=2026-04-16T16:34:28.78034822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:34:34.545293097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49434"
ts=2026-04-16T16:34:57.098914596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:45226"
ts=2026-04-16T16:34:58.782180777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:35:03.319133536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:35:04.548852501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58956"
ts=2026-04-16T16:35:18.08628644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:34770"
ts=2026-04-16T16:35:28.78355302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:35:34.552033929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53448"
ts=2026-04-16T16:35:48.088913951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:56396"
level=debug ts=2026-04-16T16:35:51.255158608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:35:58.786256952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=error ts=2026-04-16T16:36:00.247256317Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:36:00.249382707Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:36:00.251380837Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:36:00.251410532Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:36:04.554719368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55546"
ts=2026-04-16T16:36:18.091717399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:58218"
ts=2026-04-16T16:36:28.78949764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1233755e 10.130.0.74:7946"
ts=2026-04-16T16:36:34.558169607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43028"
level=debug ts=2026-04-16T16:36:36.571307944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:36:57.108858135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43798"
ts=2026-04-16T16:36:58.79145582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:37:04.560862322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60474"
ts=2026-04-16T16:37:27.109777287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39048"
ts=2026-04-16T16:37:28.793903829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:37:31.577383257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:37:31.579866525Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:37:58.79677813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1233755e 10.130.0.74:7946"
ts=2026-04-16T16:38:18.104063144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57316"
ts=2026-04-16T16:38:28.799283345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:38:29.444642481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:38:58.801757099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:39:23.711969284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:39:27.120744318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:54526"
ts=2026-04-16T16:39:28.803916645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1233755e 10.130.0.74:7946"
ts=2026-04-16T16:39:57.121899808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:59762"
ts=2026-04-16T16:39:58.807008807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1233755e 10.130.0.74:7946"
level=debug ts=2026-04-16T16:40:14.685997054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:40:27.123241726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41936"
ts=2026-04-16T16:40:28.809347767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:40:34.577233764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43018"
level=debug ts=2026-04-16T16:40:57.601048783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:40:58.810455602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:41:04.580205608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51520"
ts=2026-04-16T16:41:27.127512217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:50016"
ts=2026-04-16T16:41:28.812345237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:41:34.582666573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46828"
level=debug ts=2026-04-16T16:41:42.734408053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:41:42.736491837Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:41:58.814322852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:42:28.815665551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:42:31.307429409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:42:36.860102313Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1233755e (timeout reached)"
ts=2026-04-16T16:42:39.859302152Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1233755e has failed, no acks received"
ts=2026-04-16T16:42:49.643265435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39616"
ts=2026-04-16T16:42:49.653550342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39624"
ts=2026-04-16T16:42:56.859042141Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1233755e (timeout reached)"
ts=2026-04-16T16:42:58.818408977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:42:59.858934558Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1233755e has failed, no acks received"
ts=2026-04-16T16:42:59.860027488Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1233755e as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T16:43:28.821265535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-757f957d 10.130.0.74:7946"
level=debug ts=2026-04-16T16:43:32.261346359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:43:34.59340162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47928"
ts=2026-04-16T16:43:58.824274652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:44:25.928390627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:44:28.827381149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:44:34.599623733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57846"
ts=2026-04-16T16:44:58.829727882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-757f957d 10.130.0.74:7946"
level=debug ts=2026-04-16T16:45:15.10437503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:45:28.832840539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:45:34.605700329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60292"
ts=2026-04-16T16:45:39.632682173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:34508"
ts=2026-04-16T16:45:58.834805651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=error ts=2026-04-16T16:46:00.249623146Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:46:00.251822223Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:46:00.253901835Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:46:00.253930444Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-16T16:46:11.286667084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:46:11.288447633Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:46:27.155020559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:36916"
ts=2026-04-16T16:46:28.837030808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-757f957d 10.130.0.74:7946"
ts=2026-04-16T16:46:34.611158334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39636"
level=debug ts=2026-04-16T16:46:58.394172131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:46:58.838643586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:47:27.158720324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:35088"
ts=2026-04-16T16:47:28.84024453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-757f957d 10.130.0.74:7946"
level=debug ts=2026-04-16T16:47:39.209375324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:47:58.843499674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:48:09.64540216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:36486"
level=debug ts=2026-04-16T16:48:28.599596692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:48:28.845490675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:48:39.64797367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54002"
ts=2026-04-16T16:48:57.166870864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44556"
ts=2026-04-16T16:48:58.847803604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-757f957d 10.130.0.74:7946"
level=debug ts=2026-04-16T16:49:06.111392864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:49:09.650430063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38746"
ts=2026-04-16T16:49:27.168510498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:58248"
ts=2026-04-16T16:49:28.850083303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:49:34.626614325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34122"
ts=2026-04-16T16:49:58.851675548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:50:04.628906918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40790"
level=debug ts=2026-04-16T16:50:06.335882318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:50:06.337770998Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:50:09.655516889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:44620"
ts=2026-04-16T16:50:28.854436888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:50:39.658209638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:40556"
level=debug ts=2026-04-16T16:50:41.446834566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:50:57.175160455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:34914"
ts=2026-04-16T16:50:58.856364552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:51:04.634786161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33096"
ts=2026-04-16T16:51:27.177019142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:45932"
ts=2026-04-16T16:51:28.857894134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:51:34.324406683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:51:39.663759709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:48876"
ts=2026-04-16T16:51:58.860438867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
ts=2026-04-16T16:52:27.181599187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:57374"
ts=2026-04-16T16:52:28.863935142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=debug ts=2026-04-16T16:52:36.192092773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:52:39.669175614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54328"
ts=2026-04-16T16:52:57.182405836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:57540"
ts=2026-04-16T16:52:58.866264486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
ts=2026-04-16T16:53:04.646881199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57080"