level=info ts=2026-04-26T02:24:02.476673712Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-26T02:24:02.476733614Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-26T02:24:02.478166614Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-26T02:24:02.478497476Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-26T02:24:02.478579828Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.56 inf=eth0
level=debug ts=2026-04-26T02:24:02.479435934Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-26T02:24:02.479494031Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.56 inf=eth0
level=info ts=2026-04-26T02:24:02.479576113Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-6a6724f2
ts=2026-04-26T02:24:02.479638068Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-26T02:24:02.479649893Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.56 advertisePort=7946
level=debug ts=2026-04-26T02:24:02.479664181Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.56 advertisePort=7946
level=info ts=2026-04-26T02:24:02.479761054Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-26T02:24:02.486943069Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-26T02:24:02.486961117Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-26T02:24:02.486972752Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.213465ms
level=error ts=2026-04-26T02:24:02.486982641Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-26T02:24:02.486992059Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-26T02:24:02.490561148Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T02:24:02.490577357Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-26T02:24:02.49468614Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-26T02:24:02.494686499Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-26T02:24:02.494692068Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-26T02:24:02.494713126Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-26T02:24:02.494927188Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-26T02:24:02.494999107Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-26T02:24:02.495120571Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-26T02:24:02.495141482Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-26T02:24:02.495146151Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-26T02:24:02.495149624Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-26T02:24:02.495224197Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-26T02:24:02.495254337Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-26T02:24:02.495288505Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-26T02:24:02.495476991Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-26T02:24:02.676353187Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-26T02:24:02.676420052Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-26T02:24:03.677479338Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-26T02:24:03.799884793Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-26T02:24:03.799936788Z caller=loki.go:599 msg="Loki started" startup_time=1.383380894s
ts=2026-04-26T02:24:04.355248471Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T02:24:04.355273332Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:24:07.039168185Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T02:24:07.039193802Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-26T02:24:08.8009163Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-26T02:24:08.800966051Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-26T02:24:12.521548561Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T02:24:12.521577595Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-26T02:24:15.845246374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:24:24.56300376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58032"
ts=2026-04-26T02:24:25.757923192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:42222"
ts=2026-04-26T02:24:26.572237239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:57818"
ts=2026-04-26T02:24:28.472262168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.56:7946"
ts=2026-04-26T02:24:28.47236889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:35928"
ts=2026-04-26T02:24:28.473424025Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.55:7946: dial tcp 10.129.0.55:7946: connect: connection refused"
ts=2026-04-26T02:24:28.473545217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.54:7946"
ts=2026-04-26T02:24:28.474627564Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.46:7946: dial tcp 10.128.0.46:7946: connect: connection refused"
ts=2026-04-26T02:24:28.475032062Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.48:7946: dial tcp 10.128.0.48:7946: connect: connection refused"
level=info ts=2026-04-26T02:24:28.47504031Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=25.988042581s
level=debug ts=2026-04-26T02:24:37.398492479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:24:48.026071088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:24:48.744429367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:34264"
level=debug ts=2026-04-26T02:24:57.451120196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:25:17.28151803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59420"
level=debug ts=2026-04-26T02:25:17.942926145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:25:18.027493481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:25:41.521245802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:49150"
level=debug ts=2026-04-26T02:25:44.921772341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-26T02:25:44.923419161Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T02:25:48.029598653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:25:48.748008207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:42438"
ts=2026-04-26T02:26:11.522994721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:47708"
ts=2026-04-26T02:26:18.031490688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
level=debug ts=2026-04-26T02:26:20.855294504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:26:48.033134017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:27:18.033997477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
level=debug ts=2026-04-26T02:27:18.490153914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:27:41.527280977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:33668"
ts=2026-04-26T02:27:47.289518776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:49618"
ts=2026-04-26T02:27:48.035604998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:28:11.528151872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:53730"
ts=2026-04-26T02:28:17.291034333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:60194"
ts=2026-04-26T02:28:18.037544567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:28:18.75747264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:34644"
level=debug ts=2026-04-26T02:28:29.966382357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:28:48.039329447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:29:18.040774033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:29:18.76140875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:35938"
level=debug ts=2026-04-26T02:29:37.876077746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T02:29:48.042096079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
ts=2026-04-26T02:29:48.762960899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:57044"
ts=2026-04-26T02:30:17.296552328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41558"
ts=2026-04-26T02:30:18.044880449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
level=debug ts=2026-04-26T02:30:46.379849305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-26T02:30:46.381560322Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T02:30:47.297934761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41280"
ts=2026-04-26T02:30:48.047101447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:31:18.049379249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:31:43.524724227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:31:48.050848981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:32:18.052487559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:32:41.242131805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:32:47.304657809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:43034"
ts=2026-04-26T02:32:48.05462791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:33:17.306088054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:42106"
ts=2026-04-26T02:33:18.056646686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:33:31.053772646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:33:47.307495911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:35838"
ts=2026-04-26T02:33:48.058809024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=info ts=2026-04-26T02:34:08.801940133Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-26T02:34:08.806761693Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T02:34:08.808558687Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T02:34:08.810213831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T02:34:08.810237767Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-26T02:34:08.810245264Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-26T02:34:11.550004313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:47704"
ts=2026-04-26T02:34:18.060390272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:34:18.781051885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:60936"
level=debug ts=2026-04-26T02:34:21.542144103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:34:47.310464271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:56606"
ts=2026-04-26T02:34:48.061520461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:35:14.590914168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T02:35:14.59242731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T02:35:18.063420956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:35:48.065053394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
level=debug ts=2026-04-26T02:36:09.646434271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:36:11.555035829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:53368"
ts=2026-04-26T02:36:18.067045866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:36:18.788928444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:58262"
ts=2026-04-26T02:36:48.069025763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:36:54.991397587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:37:18.070361825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
level=debug ts=2026-04-26T02:37:40.325688049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:37:47.32104667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:37690"
ts=2026-04-26T02:37:48.072192553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:38:11.561474698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:52292"
ts=2026-04-26T02:38:17.322617064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:53380"
ts=2026-04-26T02:38:18.074000191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
level=debug ts=2026-04-26T02:38:36.023131123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:38:47.32424856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54948"
ts=2026-04-26T02:38:48.075631125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
ts=2026-04-26T02:39:11.564910963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:51286"
level=debug ts=2026-04-26T02:39:14.864819568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T02:39:14.866527619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T02:39:18.077024629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
ts=2026-04-26T02:39:47.327679908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:35236"
ts=2026-04-26T02:39:48.078570987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:39:48.803232423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:54654"
ts=2026-04-26T02:40:11.569595571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:44110"
level=debug ts=2026-04-26T02:40:13.714289056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:40:17.328663977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48516"
ts=2026-04-26T02:40:18.080123526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
ts=2026-04-26T02:40:41.570608594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:53994"
ts=2026-04-26T02:40:48.081516535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
level=debug ts=2026-04-26T02:40:54.541290929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:41:11.571837609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:52122"
ts=2026-04-26T02:41:18.083242749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-sfsgk-86de29b4 10.129.0.54:7946"
level=debug ts=2026-04-26T02:41:45.157025114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T02:41:48.084895006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297d2cdc 10.128.0.49:7946"
ts=2026-04-26T02:41:48.810822003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.49:44070"
ts=2026-04-26T02:42:11.575559085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:38906"
ts=2026-04-26T02:42:18.086245033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-ccx6l-d03ba01d 10.129.0.53:7946"
level=debug ts=2026-04-26T02:42:40.579824816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"