level=info ts=2026-05-07T02:58:38.099207578Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T02:58:38.099252484Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T02:58:38.102140747Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T02:58:38.102497405Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T02:58:38.102582724Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.54 inf=eth0
level=debug ts=2026-05-07T02:58:38.103152438Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T02:58:38.10323136Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.54 inf=eth0
level=info ts=2026-05-07T02:58:38.103497542Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-9c75f2c1
ts=2026-05-07T02:58:38.10357259Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T02:58:38.103588954Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.54 advertisePort=7946
level=debug ts=2026-05-07T02:58:38.103612264Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.54 advertisePort=7946
level=info ts=2026-05-07T02:58:38.103767386Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T02:58:38.109516588Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T02:58:38.109533773Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T02:58:38.109545925Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.780897ms
level=error ts=2026-05-07T02:58:38.109568326Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T02:58:38.109582198Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T02:58:38.112331229Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T02:58:38.112347495Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T02:58:38.120867358Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-07T02:58:38.120986258Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-07T02:58:38.121664694Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T02:58:38.12178481Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T02:58:38.121800926Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-07T02:58:38.121814661Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T02:58:38.121881814Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-07T02:58:38.121900725Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-07T02:58:38.122134201Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T02:58:38.122144621Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T02:58:38.12220103Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T02:58:38.122250197Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T02:58:38.122275143Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T02:58:38.122420981Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T02:58:38.122433386Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T02:58:38.122449197Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-07T02:58:39.123597351Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T02:58:39.234850197Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T02:58:39.234902754Z caller=loki.go:615 msg="Loki started" startup_time=1.196936046s
ts=2026-05-07T02:58:39.813195616Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T02:58:39.813219579Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T02:58:42.846993539Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T02:58:42.847017187Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T02:58:44.235635544Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T02:58:44.235684413Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T02:58:49.477607727Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T02:58:49.477637686Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T02:58:54.201529159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T02:58:57.663006085Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.52:7946: dial tcp 10.130.0.52:7946: connect: connection refused"
ts=2026-05-07T02:58:57.665146314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.79:7946"
ts=2026-05-07T02:58:57.666568401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.54:7946"
ts=2026-05-07T02:58:57.666647344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60152"
ts=2026-05-07T02:58:57.667615039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.50:7946"
ts=2026-05-07T02:58:57.668576133Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.51:7946: dial tcp 10.130.0.51:7946: connect: connection refused"
level=info ts=2026-05-07T02:58:57.668585222Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=19.558997051s
ts=2026-05-07T02:58:58.840737763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50364"
ts=2026-05-07T02:59:01.182606217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:39490"
ts=2026-05-07T02:59:01.862820755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34120"
level=debug ts=2026-05-07T02:59:15.676790023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T02:59:30.143563496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T02:59:40.864800244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T02:59:56.784879335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55480"
ts=2026-05-07T03:00:00.145481135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
level=debug ts=2026-05-07T03:00:06.120340588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T03:00:16.451426862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43886"
ts=2026-05-07T03:00:26.787965402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:50454"
ts=2026-05-07T03:00:30.1483521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:00:33.316566571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:52936"
level=debug ts=2026-05-07T03:00:33.378290213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T03:00:33.380068877Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:01:00.151144022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
level=debug ts=2026-05-07T03:01:04.778572999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T03:01:30.153390878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:01:33.319227164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:43880"
ts=2026-05-07T03:01:46.458782941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43778"
ts=2026-05-07T03:02:00.154739449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:02:03.320747107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:42520"
level=debug ts=2026-05-07T03:02:10.214697922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T03:02:30.157390982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:02:33.322341954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:47204"
ts=2026-05-07T03:02:46.462725172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43920"
ts=2026-05-07T03:02:56.801870699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46684"
ts=2026-05-07T03:03:00.16006623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:03:09.075789726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T03:03:16.463795054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34864"
ts=2026-05-07T03:03:30.16222816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:03:56.807511368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38834"
ts=2026-05-07T03:04:00.164205632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T03:04:10.82135783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T03:04:30.166093609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:05:00.168133412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:05:12.916779455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T03:05:12.918288778Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:05:16.471733572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50480"
ts=2026-05-07T03:05:30.169266049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:05:33.330910134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:42294"
ts=2026-05-07T03:05:46.473097688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:40078"
ts=2026-05-07T03:06:00.170608179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T03:06:00.185464308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:06:16.474105296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50994"
ts=2026-05-07T03:06:30.171986452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:06:33.33321693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:46260"
level=debug ts=2026-05-07T03:06:52.935844125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:06:56.826028527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:51540"
ts=2026-05-07T03:07:00.173267691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:07:16.47688444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37250"
level=debug ts=2026-05-07T03:07:29.273610412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:07:30.175274739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:07:33.33524097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:43226"
ts=2026-05-07T03:08:00.177853239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T03:08:29.712264498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:08:30.179511251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:08:33.338738227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:39382"
level=info ts=2026-05-07T03:08:44.236039029Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T03:08:44.239214586Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:08:44.240785533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:08:44.242331846Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:08:44.24235579Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T03:08:44.242367037Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T03:09:00.181273266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:09:03.339892426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:36412"
level=debug ts=2026-05-07T03:09:13.200021147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T03:09:13.201564948Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:09:30.183507392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:09:46.488032857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43988"
ts=2026-05-07T03:09:56.842073272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43170"
ts=2026-05-07T03:10:00.18555638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:10:02.780302748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:10:30.187094525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T03:10:54.410220073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:11:00.188275243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:11:30.190373136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
ts=2026-05-07T03:11:33.348472672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:48224"
ts=2026-05-07T03:11:46.498926767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43760"
level=debug ts=2026-05-07T03:11:55.356580816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:11:56.853693745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56984"
ts=2026-05-07T03:12:00.192894095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:12:26.856896466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49238"
ts=2026-05-07T03:12:30.194804913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:12:33.351994991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:43684"
level=debug ts=2026-05-07T03:12:39.363518655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:12:46.501900709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42232"
ts=2026-05-07T03:12:56.860281463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52500"
ts=2026-05-07T03:13:00.196493626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:13:24.865658011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T03:13:24.867093515Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:13:26.863546836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58462"
ts=2026-05-07T03:13:30.197419889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:13:33.355045541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:57116"
ts=2026-05-07T03:13:46.50377297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:60846"
ts=2026-05-07T03:13:56.866907381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40202"
ts=2026-05-07T03:14:00.198442447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=debug ts=2026-05-07T03:14:22.945150602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:14:30.200078731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
ts=2026-05-07T03:14:56.872687209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:47246"
ts=2026-05-07T03:15:00.202644622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
level=debug ts=2026-05-07T03:15:07.603999501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:15:16.508964442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:32808"
ts=2026-05-07T03:15:30.204642224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:15:56.878410982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42630"
ts=2026-05-07T03:16:00.206870686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
level=debug ts=2026-05-07T03:16:07.813417777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:16:16.511496708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:51392"
ts=2026-05-07T03:16:30.209159844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:16:46.649480528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:16:56.885149101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49146"
ts=2026-05-07T03:17:00.210611994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
ts=2026-05-07T03:17:30.212635835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-5kgqt-2a8e5a08 10.129.0.79:7946"
level=debug ts=2026-05-07T03:17:30.615237745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T03:17:30.617056809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:17:33.374281327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.53:56544"
ts=2026-05-07T03:17:56.890961025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36640"
ts=2026-05-07T03:18:00.214403745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:18:23.247059912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:18:30.215843652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"
level=error ts=2026-05-07T03:18:44.246957701Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:18:44.248844443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:18:44.250599451Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T03:18:44.25061809Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T03:19:00.217217944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2v88r-1a30eb16 10.130.0.50:7946"
level=debug ts=2026-05-07T03:19:08.632755894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T03:19:30.218414138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6d4c1a7e 10.130.0.53:7946"