level=info ts=2026-04-22T04:48:51.055832678Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T04:48:51.055982482Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-22T04:48:51.056013943Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-22T04:48:51.057164086Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T04:48:51.057723688Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:48:51.057844517Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.71 inf=eth0
level=info ts=2026-04-22T04:48:51.060088531Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-bb3f9d9e
ts=2026-04-22T04:48:51.060220397Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T04:48:51.060239449Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.71 advertisePort=7946
level=debug ts=2026-04-22T04:48:51.060254005Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.71 advertisePort=7946
level=info ts=2026-04-22T04:48:51.06041202Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-22T04:48:51.065358056Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T04:48:51.065379647Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-22T04:48:51.066982712Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-22T04:48:51.067371735Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:48:51.06749577Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T04:48:51.067518295Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T04:48:51.067510962Z
level=debug ts=2026-04-22T04:48:51.067536621Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-22T04:48:51.067556633Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-22T04:48:51.06757473Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T04:48:51.067586163Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T04:48:51.06758066Z
level=debug ts=2026-04-22T04:48:51.067593752Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-04-22T04:48:51.068189837Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T04:48:51.068209468Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:48:51.068233818Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.823972ms
level=error ts=2026-04-22T04:48:51.068243499Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T04:48:51.068253577Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-22T04:48:51.069214635Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T04:48:51.069307445Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.71 inf=eth0
ts=2026-04-22T04:48:51.070895005Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:51.070913322Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:48:51.088462498Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-22T04:48:51.088474893Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T04:48:51.088495132Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-22T04:48:51.088494716Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T04:48:51.088508834Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-22T04:48:51.088519716Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-22T04:48:51.08854517Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T04:48:51.088563742Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T04:48:51.088577972Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-22T04:48:51.088583969Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-22T04:48:51.088781692Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-22T04:48:51.088793436Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-22T04:48:51.088794209Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-22T04:48:51.088809738Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-22T04:48:51.088826299Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-22T04:48:51.088862536Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-22T04:48:51.088869299Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-22T04:48:51.088874842Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-22T04:48:51.088879719Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-22T04:48:51.088994972Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-22T04:48:51.089008582Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-22T04:48:51.089012015Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-22T04:48:51.089015249Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-22T04:48:51.089018747Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-22T04:48:51.089021288Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-22T04:48:51.089039195Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-22T04:48:51.089070656Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-22T04:48:51.089141638Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=105.565µs errors=false
level=info ts=2026-04-22T04:48:51.089151543Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-22T04:48:51.08959787Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=561.325µs errors=false
level=info ts=2026-04-22T04:48:51.089612836Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-22T04:48:51.089624698Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=587.933µs
level=info ts=2026-04-22T04:48:51.089647379Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-22T04:48:51.08968774Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T04:48:51.089697765Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=16.367067589s
level=info ts=2026-04-22T04:48:51.089721817Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-22T04:48:51.089801793Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-22T04:48:51.089826882Z caller=loki.go:599 msg="Loki started" startup_time=106.141297ms
level=debug ts=2026-04-22T04:48:51.089842091Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-22T04:48:51.08985159Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-22T04:48:51.089903812Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-22T04:48:52.426571041Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:52.426598624Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:48:56.090303546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:48:56.319575262Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T04:48:56.319604627Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:49:01.090520371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:49:02.300034761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-04-22T04:49:02.302699125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.71:7946"
level=info ts=2026-04-22T04:49:02.303822764Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=11.235562809s
level=warn ts=2026-04-22T04:49:06.090805845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:49:11.090804229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:49:11.436410536Z caller=logging.go:137 msg="GET /ready (503) 105.721µs"
level=warn ts=2026-04-22T04:49:16.090725499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:49:21.090650207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-22T04:49:21.090678244Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-22T04:49:21.090696758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:49:21.090706157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:49:21.435272293Z caller=logging.go:137 msg="GET /ready (503) 72.512µs"
level=warn ts=2026-04-22T04:49:26.090463084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:49:29.364070228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45826"
level=warn ts=2026-04-22T04:49:31.090647075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:49:33.925648383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52124"
level=warn ts=2026-04-22T04:49:36.090428594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:49:41.090739116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:49:41.579897415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:49:41.579993585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:49:41.585505616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:49:46.090810457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:49:46.154071434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:49:50.536538257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:49:51.066796777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:49:51.067838581Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:49:51.090221394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T04:49:51.090219985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:49:51.090245396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:49:51.090253907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:49:56.090890003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:49:59.36646229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54042"
level=warn ts=2026-04-22T04:50:01.090015834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:50:01.126157574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:50:01.136245844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:50:06.090219269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:50:07.402564629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:50:11.090277047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:50:16.090230831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:50:16.156535357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:50:17.215614575Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:50:21.090505994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:50:21.090535758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T04:50:21.090529064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:50:21.090545633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:50:26.090369114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:50:29.369544548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47322"
level=info ts=2026-04-22T04:50:30.779555208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:50:30.786972526Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:50:31.090555527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:50:33.535153299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:50:33.929803409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53376"
level=warn ts=2026-04-22T04:50:36.090246754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:50:41.090883865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:50:46.090595458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:50:46.158571365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:50:51.025590941Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:50:51.066678146Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:50:51.067750813Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-22T04:50:51.089845569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:50:51.089862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:50:51.089882091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:50:51.089890458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T04:50:54.264418658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:50:56.090158733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:51:00.466250645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:38180"
level=warn ts=2026-04-22T04:51:01.090631463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:06.090891994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:11.090277756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:16.090835946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:51:16.161156974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T04:51:21.090197681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:51:21.090234661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:51:21.090244204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:51:21.09023737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:26.090578742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:51:26.928718273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:51:29.33274728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:51:29.334725502Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T04:51:29.37540806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57820"
level=info ts=2026-04-22T04:51:29.460945951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:51:29.469266586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:51:31.089898079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:51:33.934734226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44518"
level=warn ts=2026-04-22T04:51:36.090755539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:41.09092641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:46.090808634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:51:46.164201538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T04:51:51.067523392Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T04:51:51.067528792Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:51:51.089822213Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:51:51.089848172Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:51:51.089857373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:51:51.089914014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:51:56.090610092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:51:58.344271825Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T04:51:59.378779728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59546"
ts=2026-04-22T04:52:00.469924731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60862"
level=warn ts=2026-04-22T04:52:01.090023148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:52:03.935921264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37246"
level=warn ts=2026-04-22T04:52:06.089941895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:52:11.090591162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:52:12.619755788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:52:12.628163481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:52:16.090221332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:52:16.167545189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T04:52:21.090738581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:52:21.090768866Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:52:21.090777378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:52:21.090785447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:52:21.264943307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:52:26.090394016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:52:26.238298522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T04:52:31.090275227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:52:36.090472373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:52:41.090592661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:52:46.090618678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:52:46.170884146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T04:52:51.067695959Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T04:52:51.068205859Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:52:51.090383171Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:52:51.090415031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:52:51.090426834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:52:51.090447649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:52:56.09081385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:52:58.361289931Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T04:52:59.383238552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52944"
level=info ts=2026-04-22T04:53:00.135202219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:53:00.143413056Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:53:01.090513587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:53:06.090745358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:53:11.090006589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:53:16.090533538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:53:16.173824791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T04:53:21.090488699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:53:21.090519646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:53:21.090528363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:53:21.09054846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:53:22.661610079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:53:22.661674515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="233 kB" avg_comp="233 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:53:22.665799471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T04:53:26.090861425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:53:31.090000686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:53:34.60952495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T04:53:36.090651687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:53:36.253994133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:53:36.258889333Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T04:53:41.090682164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:53:46.090238338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:53:46.175679961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:53:48.181576495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:53:51.0674997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T04:53:51.068632447Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:53:51.089846489Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T04:53:51.089870626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:53:51.089888311Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:53:51.089896952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:53:51.089918116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:53:51.090002094Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-22T04:53:51.090227503Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-22T04:53:54.406143201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:53:54.412889378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T04:53:55.446036021Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-22T04:53:56.090857986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:53:59.387662478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34738"
level=debug ts=2026-04-22T04:53:59.799974659Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T04:54:01.090599212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:04.155029668Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-04-22T04:54:04.514584065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:54:04.518742589Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T04:54:06.090262539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:08.510638948Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-22T04:54:11.090271404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:12.864939072Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-22T04:54:16.09047232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:54:16.177033044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:54:17.220341468Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-22T04:54:19.378754256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:54:21.090232005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:54:21.090263762Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:54:21.090272158Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:54:21.090275499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:21.574977682Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T04:54:25.929734905Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=warn ts=2026-04-22T04:54:26.090669671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:30.284762544Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
ts=2026-04-22T04:54:30.481849545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:35742"
level=warn ts=2026-04-22T04:54:31.090608709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:54:33.948837939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37006"
level=debug ts=2026-04-22T04:54:34.639589254Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T04:54:36.090111062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:38.908706457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T04:54:38.993968261Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T04:54:41.090864109Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:43.348586409Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T04:54:46.090565041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:54:46.178082161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:54:47.704108129Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-22T04:54:48.379327509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:54:49.000066995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:54:49.006186202Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:54:51.066823587Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:54:51.067902281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:54:51.090258444Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T04:54:51.090261645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:51.090279651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:54:51.090286827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T04:54:52.058646504Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T04:54:52.160204741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:54:52.169074188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:54:56.09040303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:54:56.413048102Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
ts=2026-04-22T04:54:59.391471377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60070"
level=debug ts=2026-04-22T04:55:00.768609491Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=debug ts=2026-04-22T04:55:00.768630962Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=17
level=warn ts=2026-04-22T04:55:01.090092085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:05.122853278Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-22T04:55:06.090521672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:09.478364254Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=warn ts=2026-04-22T04:55:11.090284125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:13.832656187Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T04:55:16.090579507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:55:16.180025115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T04:55:18.187440266Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-22T04:55:21.090116277Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:55:21.090154328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:55:21.090162987Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:55:21.090152748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:22.542677642Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=debug ts=2026-04-22T04:55:22.8104703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T04:55:26.090015739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:26.8971534Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=debug ts=2026-04-22T04:55:27.638212089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T04:55:29.394933148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45670"
level=warn ts=2026-04-22T04:55:31.090712778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:31.251992677Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-22T04:55:35.607267161Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T04:55:36.090089023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:39.965029419Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-22T04:55:39.965053647Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.4 MB" series=9
level=warn ts=2026-04-22T04:55:41.090306156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:55:41.265778244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:55:41.273658666Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T04:55:43.986220935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:55:43.989727598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:55:44.316806069Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T04:55:46.090624976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:55:46.182153182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:55:48.671683954Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-04-22T04:55:50.167862413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:55:51.066888013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:55:51.067975555Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:55:51.090278637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:55:51.090300956Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:55:51.090310136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:55:51.090311619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:53.025879685Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T04:55:56.090354384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:55:57.381286907Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
ts=2026-04-22T04:55:59.397703258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37650"
level=warn ts=2026-04-22T04:56:01.090495324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:01.736437906Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-22T04:56:06.090046356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:06.091090299Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-22T04:56:10.445262652Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-22T04:56:11.090373586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:14.800019435Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T04:56:16.090612382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:56:16.18313721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T04:56:19.155361774Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-04-22T04:56:21.090117491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:56:21.090147354Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T04:56:21.090136739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:56:21.090155911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T04:56:23.509591273Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-04-22T04:56:25.453001112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:56:25.45881259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:56:25.847135905Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T04:56:26.090221184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:27.865014193Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T04:56:31.089936218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:32.22024214Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-22T04:56:33.958000393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50256"
level=warn ts=2026-04-22T04:56:36.090028331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:36.574448924Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T04:56:38.276007095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:56:38.285233758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:56:38.285265436Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T04:56:38.285298182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:56:38.292350271Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T04:56:39.425260549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T04:56:39.427044943Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T04:56:40.929150108Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T04:56:41.090225607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:45.284416218Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-22T04:56:46.089864639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:56:46.185336621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:56:49.639592162Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=info ts=2026-04-22T04:56:50.010702044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:56:50.019752236Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T04:56:51.067207162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:56:51.068271849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:56:51.090556968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:56:51.090578016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:56:51.090586645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:56:51.090587708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:53.994502815Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=warn ts=2026-04-22T04:56:56.090502989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:56:56.139783704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:56:58.348609838Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
ts=2026-04-22T04:56:59.40188992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48796"
level=warn ts=2026-04-22T04:57:01.090272628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:02.703628824Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T04:57:06.090685432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:07.058287291Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-22T04:57:11.090286168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:11.413108368Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=debug ts=2026-04-22T04:57:11.41313107Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=21
level=info ts=2026-04-22T04:57:11.888618158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:57:11.893541875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:57:15.767657809Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=warn ts=2026-04-22T04:57:16.090586929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:57:16.187966983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T04:57:20.123563234Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-22T04:57:20.123587563Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=debug ts=2026-04-22T04:57:20.603131861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:57:21.090231431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:57:21.090255109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:57:21.090263217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:57:21.090277352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:24.477883934Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T04:57:25.027387445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:57:25.039257108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:57:26.090622017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:28.832883603Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
level=warn ts=2026-04-22T04:57:31.090835261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:33.187433394Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=debug ts=2026-04-22T04:57:33.232422439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:57:33.9616554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43268"
level=warn ts=2026-04-22T04:57:36.090245656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:37.542726037Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T04:57:41.090222312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:41.897692701Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=warn ts=2026-04-22T04:57:46.090899305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:57:46.191326574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T04:57:46.251839237Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=debug ts=2026-04-22T04:57:50.607238641Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-22T04:57:51.067427831Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T04:57:51.067432361Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=debug ts=2026-04-22T04:57:51.074786753Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T04:57:51.08988327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:57:51.089904735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:57:51.089912357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:57:51.089906701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:57:54.961581974Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T04:57:56.090471147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:57:58.473193871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:57:58.477994136Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:57:59.316364714Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
ts=2026-04-22T04:57:59.408237285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48148"
level=warn ts=2026-04-22T04:58:01.090149162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:03.671519937Z caller=checkpoint.go:378 msg="writing series" size="369 kB"
level=info ts=2026-04-22T04:58:05.640802142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:58:05.648009091Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T04:58:06.090025185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:08.026452853Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-22T04:58:08.026476613Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-04-22T04:58:11.090686309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:12.380987956Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=warn ts=2026-04-22T04:58:16.090216507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:58:16.194633954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T04:58:16.735681635Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-22T04:58:21.089881731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:58:21.089895322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:58:21.089928501Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:58:21.089937101Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T04:58:21.090982294Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 kB" series=2
level=info ts=2026-04-22T04:58:21.09689036Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-22T04:58:21.100529Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010371955s
level=debug ts=2026-04-22T04:58:24.380253573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T04:58:26.090612677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:26.353580844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T04:58:30.505687806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:36090"
level=warn ts=2026-04-22T04:58:31.090173016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:58:33.965821563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44372"
level=warn ts=2026-04-22T04:58:36.090919524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:58:41.090061025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T04:58:46.089923811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:58:46.197363751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T04:58:51.067586718Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T04:58:51.067632444Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:58:51.089914719Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T04:58:51.089944687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T04:58:51.089947881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:51.089983848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:58:51.089992522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T04:58:51.090070812Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-22T04:58:51.090339425Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-22T04:58:54.145804909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:58:54.154497026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T04:58:55.260488918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:58:55.264416682Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:58:55.445380762Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-22T04:58:56.090037869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:58:58.863665976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:58:59.80027431Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T04:59:01.090223344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:04.155750179Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T04:59:06.090223778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:08.51004692Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=debug ts=2026-04-22T04:59:09.2102252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T04:59:11.089930596Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:12.865465675Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-22T04:59:16.089905569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:59:16.199861222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T04:59:17.219647591Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-04-22T04:59:21.090141739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T04:59:21.090153557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:21.090169891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:59:21.090181257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T04:59:21.57494716Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T04:59:24.397843953Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T04:59:25.929531789Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=warn ts=2026-04-22T04:59:26.090310216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:30.284917718Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=warn ts=2026-04-22T04:59:31.089927778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:34.63922481Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T04:59:36.090126623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:59:38.903894992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T04:59:38.912647217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T04:59:38.993887109Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T04:59:41.090081251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T04:59:41.779783799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:59:41.784215165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:59:41.784235567Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T04:59:41.784256719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:59:41.787580627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:59:43.349359513Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T04:59:46.090489962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T04:59:46.20281951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T04:59:47.704097053Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-22T04:59:51.066981755Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T04:59:51.068036118Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T04:59:51.090320901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T04:59:51.090347366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T04:59:51.090355738Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T04:59:51.090362271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:52.058133652Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T04:59:52.673672592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T04:59:53.17319347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T04:59:53.178762389Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T04:59:53.215895631Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T04:59:56.09031233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T04:59:56.413571879Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T05:00:00.768514946Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=debug ts=2026-04-22T05:00:00.768538542Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-04-22T05:00:01.090791688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:00:03.973311527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34350"
level=debug ts=2026-04-22T05:00:05.122802318Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-22T05:00:06.090550106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:09.478498454Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=warn ts=2026-04-22T05:00:11.090272954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:13.832366173Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-04-22T05:00:14.858377161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:00:14.86388142Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:00:16.090464392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:00:16.205874118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:00:18.188129128Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-04-22T05:00:19.322365851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:00:19.452649359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:00:19.460019723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:00:21.090074645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:00:21.090100597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:00:21.090111335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:00:21.090131544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:22.542234753Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T05:00:26.090617352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:26.866325983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:00:26.868019995Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:00:26.897221591Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
ts=2026-04-22T05:00:29.418309126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50886"
level=warn ts=2026-04-22T05:00:31.089966746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:31.252283043Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
ts=2026-04-22T05:00:33.97408775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52304"
level=debug ts=2026-04-22T05:00:35.606923974Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T05:00:36.090883982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:39.965159936Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-04-22T05:00:39.965183733Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.4 MB" series=9
level=warn ts=2026-04-22T05:00:41.090353077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:44.316795707Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T05:00:46.090563032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:00:46.209095286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:00:46.834885493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:00:48.671584625Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-04-22T05:00:51.06742278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:00:51.067992729Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=debug ts=2026-04-22T05:00:51.068183822Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T05:00:51.068227477Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974259 n=1
level=debug ts=2026-04-22T05:00:51.068267549Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974259 n=1
level=info ts=2026-04-22T05:00:51.090275656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:00:51.090301246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:00:51.090296413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:00:51.090311795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:00:53.02618826Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:00:56.09076513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:00:57.380992337Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
ts=2026-04-22T05:00:59.421223104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48904"
ts=2026-04-22T05:01:00.517812331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45338"
level=warn ts=2026-04-22T05:01:01.090326457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:01.736405686Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
ts=2026-04-22T05:01:03.975162051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35638"
level=warn ts=2026-04-22T05:01:06.090463391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:06.090501782Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-22T05:01:08.121120873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:01:09.827531586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:01:09.834034783Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:01:10.445625813Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-22T05:01:11.090757318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:01:11.468225274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:01:11.472057757Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:01:13.648878198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:01:14.800540212Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T05:01:16.089936456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:01:16.211519407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:01:19.15579032Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=warn ts=2026-04-22T05:01:21.090510631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:01:21.09049686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:01:21.09053629Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:01:21.090549636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:01:23.51002498Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:01:26.09054619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:27.864739943Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T05:01:31.089885276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:32.219999531Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T05:01:36.090000969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:36.575170467Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T05:01:38.171928727Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:01:40.929908393Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T05:01:41.090009014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:45.284609693Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-22T05:01:46.090628793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:01:46.213607758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:01:47.427332173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:01:49.639578195Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=debug ts=2026-04-22T05:01:51.067358287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:01:51.068433647Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:01:51.09072724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:01:51.090759818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:01:51.090768205Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:01:51.090771469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:01:52.695816889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:01:52.700262409Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:01:53.993796772Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=warn ts=2026-04-22T05:01:56.090799234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:01:58.349030381Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T05:02:01.090784364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:02.703392679Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-22T05:02:04.49045299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:02:04.500356572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:02:06.090732071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:07.05843453Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-04-22T05:02:07.058458897Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=20
level=warn ts=2026-04-22T05:02:11.089855528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:11.413706993Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=debug ts=2026-04-22T05:02:14.000878717Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:02:15.76856083Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-04-22T05:02:16.090662173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:02:16.215118266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:02:20.124547122Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-22T05:02:20.124568887Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-04-22T05:02:21.090434701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:02:21.090448451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:21.090466987Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:02:21.090478597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:02:24.478600753Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:02:26.090582506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:27.700558265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:02:28.832309011Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
ts=2026-04-22T05:02:29.4287164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47512"
level=warn ts=2026-04-22T05:02:31.090229832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:33.187356055Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-22T05:02:36.089940385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:37.54258501Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:02:41.090777503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:02:41.897976214Z caller=checkpoint.go:378 msg="writing series" size="302 kB"
level=debug ts=2026-04-22T05:02:43.706624974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:02:46.090754528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:02:46.217056522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:02:46.252662965Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=debug ts=2026-04-22T05:02:50.606828269Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-22T05:02:51.067578359Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:02:51.067618517Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:02:51.089909052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:02:51.089937892Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:02:51.089946658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:02:51.089941559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:02:52.586159405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:02:52.590329687Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:02:54.962326773Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T05:02:56.090535973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:02:57.219248742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:02:57.227767289Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:02:57.227788772Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:02:57.22781592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:02:57.233825606Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:02:59.316232952Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
ts=2026-04-22T05:03:00.528108203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55276"
level=warn ts=2026-04-22T05:03:01.090786179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:03.671673688Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=debug ts=2026-04-22T05:03:03.671700561Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-22T05:03:06.09064112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:08.026078467Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-22T05:03:11.090482642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:12.380978651Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-22T05:03:13.034173072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:03:13.043147406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:03:15.155695922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:03:16.090790046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:03:16.220165249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:03:16.736110638Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T05:03:16.76394267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:03:21.090678905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:03:21.090690752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:21.090706972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-22T05:03:21.090702225Z caller=checkpoint.go:390 msg="flushing series" totalSize="241 kB" series=3
level=info ts=2026-04-22T05:03:21.090716254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:03:21.095621821Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-22T05:03:21.097004608Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006727504s
level=warn ts=2026-04-22T05:03:26.090696248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:03:31.090773367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:03:36.090599903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:03:39.0510677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:03:39.05697495Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:03:41.090214258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:45.289130258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:03:46.090664551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:03:46.223025975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T05:03:51.067675972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:03:51.06814322Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:03:51.090413657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:03:51.090423186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:51.090433297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:03:51.090427762Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:03:51.090444544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:03:51.090562839Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-22T05:03:51.090744097Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-22T05:03:51.586709724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:03:51.594810758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:03:55.445879035Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-22T05:03:56.09065376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:03:59.800998972Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=warn ts=2026-04-22T05:04:01.090155918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:02.737871721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:04:02.739441962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:04:04.155271583Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T05:04:06.089868007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:08.51042516Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=warn ts=2026-04-22T05:04:11.09026725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:12.493028267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:04:12.86533004Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-22T05:04:16.09084794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:04:16.225719025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:04:17.220219431Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-04-22T05:04:21.090428123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:04:21.090447488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:21.090467396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:04:21.090476716Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:04:21.574974844Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T05:04:25.930118413Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-22T05:04:26.090007767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:04:29.437978952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50496"
level=debug ts=2026-04-22T05:04:30.285092636Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
ts=2026-04-22T05:04:30.534876477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46124"
level=warn ts=2026-04-22T05:04:31.090603191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:04:33.140775839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:04:33.148944855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:04:34.640062197Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:04:36.090168422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:37.750918221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:04:38.994579502Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:04:41.090100786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:04:42.779962173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:04:42.786786043Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:04:43.349344793Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T05:04:46.089860352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:04:46.228302204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:04:47.704637382Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-22T05:04:50.565406385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:04:51.067180007Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:04:51.068233681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:04:51.090524301Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:04:51.090544881Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:04:51.090553382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:04:51.090552234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:52.058508187Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-22T05:04:56.090269031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:04:56.413601823Z caller=checkpoint.go:378 msg="writing series" size="1.9 kB"
level=debug ts=2026-04-22T05:05:00.768723303Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=debug ts=2026-04-22T05:05:00.768745921Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-04-22T05:05:01.090881058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:03.190209218Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:05:05.123239069Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-22T05:05:06.090067218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:09.479065479Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=warn ts=2026-04-22T05:05:11.090831944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:13.833732753Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T05:05:16.090144916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:05:16.231525606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:05:18.188202986Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-04-22T05:05:21.090554979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:05:21.090587498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:05:21.090587423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:05:21.09059865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:05:22.543425314Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-22T05:05:25.925844045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:05:25.935987044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:05:26.090719742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:26.897700368Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
ts=2026-04-22T05:05:29.443487312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50982"
level=warn ts=2026-04-22T05:05:31.090460224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:31.252840911Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-22T05:05:32.472061363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:05:32.476589462Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:05:33.998565095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60976"
level=debug ts=2026-04-22T05:05:35.60704442Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T05:05:36.090667516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:36.575033226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:05:39.966456766Z caller=checkpoint.go:378 msg="writing series" size="6.8 MB"
level=debug ts=2026-04-22T05:05:39.966479141Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.7 MB" series=9
level=warn ts=2026-04-22T05:05:41.090258962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:44.317313866Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T05:05:46.090870936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:05:46.234320107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:05:48.672409457Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=debug ts=2026-04-22T05:05:51.066410824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:05:51.068280442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:05:51.090588668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:05:51.090611396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:05:51.090602704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:05:51.090619178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:05:52.934795083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:05:53.02699869Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:05:56.090838656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:05:57.381513878Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-04-22T05:06:01.09085893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:01.736189159Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-22T05:06:05.803008967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:06:06.090732509Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T05:06:06.090741926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:10.445679384Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-22T05:06:11.090082924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:14.8012903Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T05:06:16.090729815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:06:16.237181308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:06:19.155333626Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=info ts=2026-04-22T05:06:21.089819305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:06:21.089862873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:06:21.08987463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:06:21.08987627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:06:21.17115784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:06:21.180200761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:06:23.510802365Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:06:26.089877046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:27.865019818Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-22T05:06:30.978314437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:06:30.983773899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:06:30.983794517Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:06:30.983817399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:06:30.987155322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:06:31.090601714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:32.220084419Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-22T05:06:36.090058027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:36.574826544Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T05:06:39.505966754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:06:40.929682004Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T05:06:41.090650261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:06:42.725013047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:06:42.730772544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:06:45.284697135Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-22T05:06:46.090225207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:06:46.239612116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:06:47.116389387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:06:49.639391774Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=debug ts=2026-04-22T05:06:49.639414899Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=debug ts=2026-04-22T05:06:51.066642245Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:06:51.067709376Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-22T05:06:51.090065225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:06:51.090069133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:06:51.090087158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:06:51.090095156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:06:53.994285307Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-22T05:06:56.090699037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:06:58.349590718Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T05:07:01.090808643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:02.704150384Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T05:07:06.089872224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:07.05876349Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-22T05:07:07.525177238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:07:10.50344866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:07:10.51344871Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:07:11.090003212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:11.41360762Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-04-22T05:07:11.61699524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:07:11.620992732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:07:15.768833646Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=warn ts=2026-04-22T05:07:16.090367192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:07:16.242668739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:07:20.124835293Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-22T05:07:20.124857914Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=info ts=2026-04-22T05:07:21.090811971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:07:21.090843078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:07:21.090854121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:07:21.090850126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:24.478018159Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:07:26.090627335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:28.832827614Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
level=warn ts=2026-04-22T05:07:31.090551287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:33.187699474Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=debug ts=2026-04-22T05:07:34.603607653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:07:36.090879577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:37.543349591Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:07:41.090759108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:41.897663602Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=debug ts=2026-04-22T05:07:43.067890338Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:07:46.090511748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:07:46.245039884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:07:46.252765619Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=debug ts=2026-04-22T05:07:50.607703766Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T05:07:51.067445663Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:07:51.067929871Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:07:51.090252613Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:07:51.090263944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:51.090277066Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:07:51.090295504Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:07:53.810842297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:07:53.81651453Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:07:54.96188146Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T05:07:56.09066913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:07:59.317591452Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-22T05:08:01.090449391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:08:03.324223949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:08:03.332355289Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:08:03.671699651Z caller=checkpoint.go:378 msg="writing series" size="350 kB"
level=debug ts=2026-04-22T05:08:03.671723851Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-04-22T05:08:06.090898527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:08:08.026946503Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-22T05:08:11.089853583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:08:12.381641495Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-22T05:08:14.358298077Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:08:15.073886412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:08:15.075596866Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T05:08:16.090809222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:08:16.246485034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:08:16.736613835Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:08:21.090216793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:08:21.090247047Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:08:21.090256041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:08:21.090274338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:08:21.09128153Z caller=checkpoint.go:390 msg="flushing series" totalSize="244 kB" series=3
level=info ts=2026-04-22T05:08:21.097709012Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-22T05:08:21.099161022Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008474512s
level=warn ts=2026-04-22T05:08:26.090129869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:08:30.554157064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:44490"
level=warn ts=2026-04-22T05:08:31.090130204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:08:34.014732779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49146"
level=warn ts=2026-04-22T05:08:36.090436906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:08:38.981591478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:08:41.090320793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:08:46.032982289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:08:46.040468836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:08:46.089892842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:08:46.248370199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:08:48.629299254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:08:48.638443266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:08:51.066510287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:08:51.067574353Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:08:51.0898819Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T05:08:51.089901833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:08:51.089906359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:08:51.089925676Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:08:51.08993468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:08:51.090030044Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-22T05:08:51.090273144Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-22T05:08:55.445548324Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-22T05:08:56.090433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:08:59.457892705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48296"
level=debug ts=2026-04-22T05:08:59.800239943Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=warn ts=2026-04-22T05:09:01.09026494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:04.155592833Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T05:09:06.090866657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:08.509978398Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=debug ts=2026-04-22T05:09:09.460446196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:09:11.089998162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:11.624060774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:09:12.864666758Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T05:09:16.090366031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:09:16.250751379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:09:17.220052669Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-04-22T05:09:21.089798638Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:09:21.089830182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:09:21.089838744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:09:21.089857266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:21.574975508Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T05:09:25.929701285Z caller=checkpoint.go:378 msg="writing series" size="243 kB"
level=warn ts=2026-04-22T05:09:26.090234539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:09:29.461192713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48996"
level=debug ts=2026-04-22T05:09:30.283923257Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=warn ts=2026-04-22T05:09:31.090406327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:09:32.530072228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:09:32.536134392Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:09:34.639675319Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:09:36.090205131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:38.994501297Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=debug ts=2026-04-22T05:09:38.994526051Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-22T05:09:41.090622663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:43.349315059Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=debug ts=2026-04-22T05:09:45.371014065Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:09:46.09031733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:09:46.253787715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:09:47.704021398Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-22T05:09:47.799310328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:09:47.807168023Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:09:47.807192946Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:09:47.80721688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:09:47.813143256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:09:51.067070183Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:09:51.068152841Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:09:51.090448201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:09:51.090459102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:51.090474355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:09:51.090482982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:09:52.058173647Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-22T05:09:56.090082944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:09:56.413196578Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-04-22T05:10:00.768164587Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=warn ts=2026-04-22T05:10:01.090845323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:02.642113884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:10:04.79700271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:10:04.80328322Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:10:05.123239314Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-22T05:10:06.090135463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:09.478356294Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=warn ts=2026-04-22T05:10:11.090216231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:13.832794674Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T05:10:16.090561306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:10:16.256209175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:10:17.196304208Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:10:18.188017182Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-22T05:10:20.747450345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:10:20.752434451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:10:21.090745528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:10:21.090772295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:10:21.09078318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:10:21.09079701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:22.542405392Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-22T05:10:26.089866104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:26.897468668Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=debug ts=2026-04-22T05:10:26.897492322Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-04-22T05:10:29.466281809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39386"
level=warn ts=2026-04-22T05:10:31.090245117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:31.25192591Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-22T05:10:32.628530748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:10:32.639689691Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:10:35.607060066Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T05:10:36.091012308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:39.967741831Z caller=checkpoint.go:378 msg="writing series" size="7.5 MB"
level=debug ts=2026-04-22T05:10:39.9677654Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.6 MB" series=3
level=warn ts=2026-04-22T05:10:41.09057963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:44.316328621Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T05:10:46.090205789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:10:46.25882489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:10:48.671202676Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-22T05:10:51.067170115Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:10:51.068213534Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:10:51.090549774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:10:51.090585123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:10:51.090597498Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:10:51.090596374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:51.59319671Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:10:53.026756131Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:10:56.090411036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:10:57.380996352Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
ts=2026-04-22T05:10:59.46842551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60490"
level=warn ts=2026-04-22T05:11:01.090820303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:01.100765935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:11:01.735937505Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-22T05:11:06.090376892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:06.090382005Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-22T05:11:10.445156424Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-22T05:11:11.090004447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:14.800623825Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T05:11:16.090679295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:11:16.262177867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:11:17.601498721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:11:17.611809418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:11:17.933717404Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:11:19.15541939Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=info ts=2026-04-22T05:11:19.598003268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:11:19.60213246Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:11:21.090759702Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:11:21.090792259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:11:21.090800942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:11:21.090795786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:23.510066725Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:11:26.090383853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:27.865115395Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-22T05:11:30.565835029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55016"
level=warn ts=2026-04-22T05:11:31.090322606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:32.220398591Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-22T05:11:36.090188064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:36.574476073Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T05:11:40.929803223Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T05:11:41.090679106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:44.920229461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:11:45.284699969Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-22T05:11:46.090493781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:46.126708835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T05:11:46.264231737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:11:49.638762927Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=debug ts=2026-04-22T05:11:51.06713394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:11:51.06818802Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:11:51.090547733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:11:51.090567548Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:11:51.090560456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:11:51.090576558Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:11:53.994268387Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-22T05:11:56.090382864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:11:58.348995171Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T05:12:01.090366138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:12:01.959459983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:12:01.965105148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:12:02.70333272Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T05:12:02.703356657Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=19
ts=2026-04-22T05:12:04.034318495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50676"
level=warn ts=2026-04-22T05:12:06.090916376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:07.058922702Z caller=checkpoint.go:378 msg="writing series" size="284 kB"
level=warn ts=2026-04-22T05:12:11.089922839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:11.414015922Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=info ts=2026-04-22T05:12:12.213850383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:12:12.224491717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:12:15.768581594Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=warn ts=2026-04-22T05:12:16.090392311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:12:16.267260458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:12:19.620291004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:12:20.123617722Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-22T05:12:20.123642129Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=4
level=info ts=2026-04-22T05:12:21.090512374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:12:21.09052343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:21.090541838Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:12:21.090550877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:12:24.47815891Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:12:26.090268578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:28.832255893Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
level=warn ts=2026-04-22T05:12:31.090756289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:33.187981138Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-22T05:12:36.090675487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:37.542650693Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:12:41.090149392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:41.897487224Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-04-22T05:12:42.706899235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:12:42.708688836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:12:45.427420726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:12:46.090616549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:46.252170143Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
ts=2026-04-22T05:12:46.270187334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:12:49.108186737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:12:49.112194652Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:12:49.112214772Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:12:49.112237426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:12:49.115508044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:12:50.607210402Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=debug ts=2026-04-22T05:12:51.066311414Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:12:51.06843835Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:12:51.090758085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:12:51.090760649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:51.090780521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:12:51.090789028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:12:52.459688598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:12:52.466254443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:12:54.961584529Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T05:12:56.089981443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:12:59.316645181Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=debug ts=2026-04-22T05:12:59.316669775Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-22T05:12:59.392812075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:12:59.397226274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:12:59.47442323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60218"
level=warn ts=2026-04-22T05:13:01.090553149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:13:03.67213579Z caller=checkpoint.go:378 msg="writing series" size="468 kB"
ts=2026-04-22T05:13:04.039219547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43738"
level=warn ts=2026-04-22T05:13:06.090402581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:13:08.026401786Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-04-22T05:13:11.090871366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:13:12.381386752Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-22T05:13:16.090447978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:13:16.272290277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:13:16.736356636Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T05:13:18.170202757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:13:21.090292635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:13:21.09034076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:13:21.090349791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:13:21.090305139Z caller=checkpoint.go:390 msg="flushing series" totalSize="719 kB" series=4
level=warn ts=2026-04-22T05:13:21.090349534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:13:21.096708765Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-22T05:13:21.098069919Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007867866s
level=debug ts=2026-04-22T05:13:21.780766857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:13:26.090218879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:13:31.090839555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:13:31.212199616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:13:31.216297234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:13:34.040667843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34438"
level=warn ts=2026-04-22T05:13:36.090882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:13:39.2803378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:13:39.287184654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:13:41.090648795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:13:46.08985854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:13:46.273508573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:13:50.66101336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:13:51.067726752Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:13:51.068261033Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:13:51.090612406Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T05:13:51.090622385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:13:51.090629457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:13:51.090651462Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:13:51.09066221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:13:51.090764646Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-22T05:13:51.091019439Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-22T05:13:55.446454207Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-22T05:13:56.090132845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:13:59.801247669Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-22T05:14:00.576926813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48508"
level=warn ts=2026-04-22T05:14:01.090257667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:04.155590763Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T05:14:06.090406302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:08.510664962Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-22T05:14:11.090554146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:12.865679333Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-22T05:14:13.706105553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:14:13.710860331Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:14:16.090716453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:14:16.275714134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:14:16.934629556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:14:17.220517182Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=info ts=2026-04-22T05:14:21.090130843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:14:21.090160129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:14:21.09016887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:14:21.090160603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:21.575716548Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T05:14:23.235395024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:14:25.930792021Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=warn ts=2026-04-22T05:14:26.089920851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:30.285234664Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-22T05:14:31.089897862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:14:34.044511678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56098"
level=debug ts=2026-04-22T05:14:34.639767642Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T05:14:36.09078363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:14:37.827008156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:14:37.83575367Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:14:38.995045439Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:14:41.090783765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:43.349549412Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T05:14:46.090388164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:14:46.278796071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:14:47.282791523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:14:47.704888584Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-04-22T05:14:51.067453707Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:14:51.067898454Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:14:51.090395811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:14:51.090409683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:51.090427512Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:14:51.090438547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:14:52.059735708Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-22T05:14:56.090944643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:14:56.414428628Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
ts=2026-04-22T05:14:59.485675374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39102"
level=debug ts=2026-04-22T05:15:00.769668872Z caller=checkpoint.go:378 msg="writing series" size="497 kB"
level=debug ts=2026-04-22T05:15:00.769692194Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=17
level=warn ts=2026-04-22T05:15:01.09053202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:15:04.04809329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37954"
level=debug ts=2026-04-22T05:15:05.124241579Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-22T05:15:06.090835322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:15:08.043392402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:15:08.050554225Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:15:09.479009575Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=debug ts=2026-04-22T05:15:10.468289362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:15:11.0906938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:13.833226175Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T05:15:16.090552597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:15:16.282007579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:15:17.211028294Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:15:17.211112065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:15:17.216396866Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:15:18.188493736Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=info ts=2026-04-22T05:15:21.090526399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:15:21.090560259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:15:21.090571086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:15:21.090567826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:15:21.938907569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:15:21.947324747Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:15:22.543402967Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-04-22T05:15:26.090198267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:26.89798135Z caller=checkpoint.go:378 msg="writing series" size="371 kB"
level=debug ts=2026-04-22T05:15:26.898015153Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
ts=2026-04-22T05:15:29.488679105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50198"
level=warn ts=2026-04-22T05:15:31.090346363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:31.252646185Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-22T05:15:35.608015814Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=warn ts=2026-04-22T05:15:36.090483917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:15:36.19881725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:15:36.207493273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:15:39.967733887Z caller=checkpoint.go:378 msg="writing series" size="8.2 MB"
level=debug ts=2026-04-22T05:15:39.967757431Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=3
level=warn ts=2026-04-22T05:15:41.089896964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:44.317074328Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-22T05:15:46.090527391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:15:46.284389728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:15:48.67347667Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=info ts=2026-04-22T05:15:51.067871055Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:15:51.068126205Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T05:15:51.068186028Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974260 n=1
level=debug ts=2026-04-22T05:15:51.068229949Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974260 n=1
level=debug ts=2026-04-22T05:15:51.068407277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=debug ts=2026-04-22T05:15:51.088769779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:15:51.09086241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:15:51.090887037Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:15:51.090895802Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:15:51.090888226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:53.026856442Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:15:56.090620707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:15:57.381732359Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T05:16:00.695780457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:16:00.699647375Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:16:01.09081205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:01.736648482Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=debug ts=2026-04-22T05:16:03.489897898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:16:06.090473475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:06.091535901Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-22T05:16:10.44633557Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-22T05:16:11.090465543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:14.80067772Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=info ts=2026-04-22T05:16:15.313072161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:16:15.321114653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:16:16.090235804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:16:16.286294356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:16:19.156164363Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-04-22T05:16:20.913732041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:16:20.920748398Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:16:20.920770281Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:16:20.92079744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:16:20.926885508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:16:21.0899959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:16:21.089995966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:16:21.090025314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:16:21.090034694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:16:23.510429278Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=debug ts=2026-04-22T05:16:24.575258559Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:16:26.090444162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:27.866030334Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-22T05:16:29.492608166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52056"
level=warn ts=2026-04-22T05:16:31.09074453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:32.22069187Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-22T05:16:36.090003164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:36.5752413Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T05:16:37.442831347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:16:37.45106389Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:16:40.930184347Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-22T05:16:41.090099165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:45.285139371Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-04-22T05:16:46.090710503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:16:46.288021907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:16:47.237126115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:16:47.238732467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:16:49.640328808Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=debug ts=2026-04-22T05:16:49.640355534Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=info ts=2026-04-22T05:16:51.067421862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:16:51.067427558Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:16:51.089803237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:16:51.089834523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:16:51.089843897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:16:51.089861155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:53.994460116Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=warn ts=2026-04-22T05:16:56.09069954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:16:56.967883572Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:16:58.100511079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:16:58.105268707Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:16:58.349981032Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-22T05:17:00.59198874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46596"
level=warn ts=2026-04-22T05:17:01.090112684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:17:01.773185912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:17:01.780912283Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:17:02.704782168Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
ts=2026-04-22T05:17:04.058646305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53760"
level=warn ts=2026-04-22T05:17:06.090131881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:07.059267465Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-04-22T05:17:10.865290193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:17:10.871379243Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:17:11.090114665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:11.414279781Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=debug ts=2026-04-22T05:17:15.768840967Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T05:17:16.090558996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:17:16.290101863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:17:20.124445271Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-22T05:17:20.124469062Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=7
level=info ts=2026-04-22T05:17:21.089907237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:17:21.089937399Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:17:21.089945607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:17:21.089988715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:23.299641131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:17:24.478803642Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:17:26.090102441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:26.678139553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:17:28.834024016Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
ts=2026-04-22T05:17:30.594660641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:35700"
level=warn ts=2026-04-22T05:17:31.09054091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:33.18798697Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-22T05:17:36.089992031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:37.543027457Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-22T05:17:39.76089802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:17:39.767776811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:17:41.090183338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:41.899667385Z caller=checkpoint.go:378 msg="writing series" size="3.2 MB"
level=debug ts=2026-04-22T05:17:41.899691355Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=5
level=warn ts=2026-04-22T05:17:46.090455014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:46.253307204Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
ts=2026-04-22T05:17:46.291504894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:17:50.607460168Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-04-22T05:17:50.81409592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:17:50.82472971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:17:51.066635471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:17:51.067710315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:17:51.090210226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:17:51.090235383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:17:51.090243914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:17:51.090260606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:17:53.888362465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:17:53.893581501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:17:53.918759505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:17:54.962822679Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T05:17:56.089913348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:17:59.317854526Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
ts=2026-04-22T05:17:59.501580863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47072"
level=warn ts=2026-04-22T05:18:01.090842002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:18:03.67304476Z caller=checkpoint.go:378 msg="writing series" size="550 kB"
level=warn ts=2026-04-22T05:18:06.091075271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:18:08.027081199Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=debug ts=2026-04-22T05:18:08.027115231Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-04-22T05:18:10.247787387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:18:11.09059825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:18:12.381430989Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-22T05:18:16.09082259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:18:16.292549719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:18:16.736254352Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-22T05:18:21.090387824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:18:21.090445404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:18:21.090468318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:18:21.090479073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:18:21.092259978Z caller=checkpoint.go:390 msg="flushing series" totalSize="34 kB" series=2
level=debug ts=2026-04-22T05:18:21.106706371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:18:21.199846554Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-22T05:18:21.202513115Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.111568738s
level=warn ts=2026-04-22T05:18:26.090303103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:18:26.146419379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:18:26.151863664Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:18:29.504245527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52788"
ts=2026-04-22T05:18:30.600334424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:36770"
level=warn ts=2026-04-22T05:18:31.090675301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:18:36.090557582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:18:41.090890784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:18:41.635323785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:18:41.643354083Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:18:45.618601464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:18:46.091032721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:18:46.295753977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:18:51.067240249Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:18:51.068309942Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-22T05:18:51.091795366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:18:51.091857641Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:18:51.09250461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:18:51.092532374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:18:51.092553417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:18:51.093661552Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-22T05:18:51.095063905Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-04-22T05:18:52.366225153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:18:52.379972783Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:18:55.187092786Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-22T05:18:56.091017332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:18:57.342892702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:18:59.276075074Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=warn ts=2026-04-22T05:19:01.091133364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:03.367510715Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T05:19:06.090289192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:07.457933319Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-22T05:19:11.090037603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:11.549266342Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=debug ts=2026-04-22T05:19:14.126414524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:19:15.596598166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:19:15.603132557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:19:15.60316453Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:19:15.603193561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:19:15.607070735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:19:15.640245815Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=warn ts=2026-04-22T05:19:16.094923013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:19:16.297222052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:19:19.730301925Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-22T05:19:21.090038177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:19:21.090992719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:19:21.091025393Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:19:21.091036852Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:19:23.822006592Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=warn ts=2026-04-22T05:19:26.090699702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:27.913015642Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
ts=2026-04-22T05:19:29.509018395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44264"
ts=2026-04-22T05:19:30.605214349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54922"
level=info ts=2026-04-22T05:19:30.998367477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:19:31.004675447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:19:31.090726046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:32.003290553Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T05:19:32.470975225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:19:32.477851398Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T05:19:35.766883864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:19:35.775805752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:19:36.09079616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:36.094944377Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=debug ts=2026-04-22T05:19:40.185348121Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-22T05:19:40.620083555Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:19:41.090637167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:44.2757982Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-22T05:19:44.275823238Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-04-22T05:19:46.090415501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:19:46.299201781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:19:48.36664645Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T05:19:50.39571376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:19:51.066684616Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:19:51.067725537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:19:51.090212303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:19:51.090241105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:19:51.090234268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:19:51.090251924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:19:52.457862984Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T05:19:56.090340113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:19:56.549583932Z caller=checkpoint.go:378 msg="writing series" size="382 kB"
ts=2026-04-22T05:19:59.51227259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40770"
level=debug ts=2026-04-22T05:20:00.640310809Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-22T05:20:01.090115511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:20:03.983071627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:20:03.990677244Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:20:04.074002592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47690"
level=debug ts=2026-04-22T05:20:04.731351304Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=warn ts=2026-04-22T05:20:06.089982883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:06.165277638Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:20:08.821223445Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-22T05:20:08.821256199Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T05:20:11.090309991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:12.912366569Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=warn ts=2026-04-22T05:20:16.089932043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:20:16.300783888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=info ts=2026-04-22T05:20:16.477021771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:20:16.490472549Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:20:17.003261302Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=info ts=2026-04-22T05:20:21.090738592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:20:21.090789265Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:20:21.090798005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:20:21.090833985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:21.095075958Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=debug ts=2026-04-22T05:20:25.185880258Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T05:20:26.089993453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:29.276305125Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T05:20:31.090868723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:20:31.463647121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:20:31.469825653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:20:32.181536585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:20:32.762890694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:20:32.764469617Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:20:33.380103004Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-04-22T05:20:33.380135575Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.8 MB" series=6
ts=2026-04-22T05:20:34.076031723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56818"
level=warn ts=2026-04-22T05:20:36.090895592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:37.457757013Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-22T05:20:41.090106885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:41.549366045Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=debug ts=2026-04-22T05:20:45.639407533Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:20:46.090521207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:20:46.302280811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:20:49.731135078Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T05:20:51.067518108Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:20:51.067998671Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:20:51.090538556Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:20:51.090575367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:20:51.090586149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:20:51.090615181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:53.821776597Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=warn ts=2026-04-22T05:20:56.090632898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:20:57.91229486Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
ts=2026-04-22T05:20:59.517821659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56740"
level=debug ts=2026-04-22T05:21:00.53201561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:21:00.694392729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:21:00.700897198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:21:01.089211488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:21:01.090329569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-22T05:21:01.097816728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:21:02.003977187Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T05:21:06.090857591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:06.09401984Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-22T05:21:10.185160956Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T05:21:11.090475186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:14.276435218Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=warn ts=2026-04-22T05:21:16.089874331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:21:16.304617003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:21:18.366728569Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=debug ts=2026-04-22T05:21:18.366754602Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-22T05:21:21.090746933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:21:21.090781974Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:21:21.090794633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:21:21.090792962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:22.458045763Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-22T05:21:25.012937962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:21:26.090070599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:26.549006329Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-04-22T05:21:27.004200011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:21:27.011580041Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:21:27.195316211Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T05:21:30.612150382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47590"
level=debug ts=2026-04-22T05:21:30.639708365Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T05:21:31.090660558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:34.730449798Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T05:21:36.090453014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:38.82141138Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:21:41.089944821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:42.912135611Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-04-22T05:21:44.414533316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:21:44.42039433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:21:46.090877056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:21:46.307441661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:21:47.003605687Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-22T05:21:49.424760073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:21:49.433916954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:21:51.066798749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:21:51.067876743Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:21:51.090392067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:21:51.090416174Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:21:51.090426457Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:21:51.090450099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:51.094643777Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-22T05:21:55.185431861Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-22T05:21:56.090558852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:21:59.275770025Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-22T05:21:59.521775428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55430"
level=debug ts=2026-04-22T05:21:59.931265705Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:22:01.090125357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:03.367014188Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T05:22:03.600440344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:22:06.090558059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:07.458039291Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=warn ts=2026-04-22T05:22:11.090238594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:11.548835033Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-04-22T05:22:11.548859043Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-04-22T05:22:15.639539343Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=warn ts=2026-04-22T05:22:16.090753095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:22:16.310276664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:22:17.92010796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:22:17.927785876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:22:17.927810943Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:22:17.927836812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:22:17.932512151Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:22:19.731115538Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-22T05:22:21.090605771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:22:21.090641847Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:22:21.090654446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:22:21.090684588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:23.822273298Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-22T05:22:23.822299647Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=3
level=warn ts=2026-04-22T05:22:26.090649884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:27.912382061Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-22T05:22:29.524967634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53090"
level=info ts=2026-04-22T05:22:30.111667059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="50 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:22:30.118306258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:22:31.090893055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:32.003977255Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
ts=2026-04-22T05:22:34.086253224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46076"
level=debug ts=2026-04-22T05:22:35.424101237Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:22:36.09049585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:36.094716682Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-04-22T05:22:37.303067822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:22:37.310259857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:22:40.185041756Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:22:41.089866043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:22:41.569109816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:22:41.576396631Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:22:41.576425059Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:22:41.576457214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:22:41.582602635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:22:44.280036717Z caller=checkpoint.go:378 msg="writing series" size="6.6 MB"
level=debug ts=2026-04-22T05:22:44.280065547Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.9 MB" series=5
level=warn ts=2026-04-22T05:22:46.090566171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:22:46.312918396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:22:48.367341711Z caller=checkpoint.go:378 msg="writing series" size="384 kB"
level=debug ts=2026-04-22T05:22:51.067064282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:22:51.068093761Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:22:51.09054354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:22:51.09057975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:22:51.090584095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:22:51.090591964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:22:52.458191886Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=debug ts=2026-04-22T05:22:53.880569219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:22:56.090853762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:22:56.54859027Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-22T05:22:59.527942558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56958"
level=debug ts=2026-04-22T05:23:00.640465055Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=warn ts=2026-04-22T05:23:01.09060872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:23:01.420311312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:23:01.430161737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:23:04.731035019Z caller=checkpoint.go:378 msg="writing series" size="586 kB"
level=debug ts=2026-04-22T05:23:04.731059794Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-22T05:23:06.090724313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:23:08.822105603Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=debug ts=2026-04-22T05:23:10.881173306Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:23:11.090121821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:23:12.913081052Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-22T05:23:14.452685291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:23:14.460714529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:23:16.090103626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:23:16.315794447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:23:17.003276571Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:23:21.090516375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:23:21.090546379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:23:21.090554801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:23:21.09057233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:23:21.094726775Z caller=checkpoint.go:390 msg="flushing series" totalSize="289 kB" series=3
level=info ts=2026-04-22T05:23:21.098064103Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-22T05:23:21.102137006Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008302349s
level=info ts=2026-04-22T05:23:25.58381391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="50 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:23:25.591066798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:23:26.09065521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:23:26.863633559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:23:26.871506445Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:23:31.089931274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:23:36.090795914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:23:41.089857134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:23:45.464610252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:23:46.090409356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:23:46.318931618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:23:51.0673163Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:23:51.068373402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:23:51.089730995Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:23:51.089787793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:23:51.089817931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:23:51.089826391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:23:51.089872318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:23:51.089875999Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-22T05:23:51.090097889Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-22T05:23:52.477422694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:23:55.181346313Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-04-22T05:23:55.942315015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:23:55.950635115Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:23:56.090536264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:23:59.272617728Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-22T05:24:00.626307613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39608"
level=warn ts=2026-04-22T05:24:01.090843957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:03.362941421Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
ts=2026-04-22T05:24:04.09373335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34124"
level=warn ts=2026-04-22T05:24:06.090035389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:07.454046779Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=info ts=2026-04-22T05:24:08.363148175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="50 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:24:08.369706326Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:24:11.090634743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:24:11.196863077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:24:11.204135344Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:24:11.545298639Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-22T05:24:15.636371047Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=warn ts=2026-04-22T05:24:16.090481932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:16.103689157Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T05:24:16.322021403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:24:19.727363853Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-22T05:24:21.089982823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:24:21.090013303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:24:21.090008655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:24:21.090021975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:24:23.818161325Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=warn ts=2026-04-22T05:24:26.090816743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:27.539725071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:24:27.541238094Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:24:27.909383823Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-22T05:24:31.09044961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:31.999890351Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T05:24:36.090844861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:36.090898854Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=debug ts=2026-04-22T05:24:40.181676494Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T05:24:41.089865515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:44.272829798Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-22T05:24:44.272856241Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-04-22T05:24:45.085262918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:24:45.960093132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:24:45.966467931Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:24:46.090990651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:24:46.323881711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:24:48.363678945Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T05:24:51.067619966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:24:51.067639778Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:24:51.090149528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:24:51.090176635Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:24:51.090186268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:24:51.090202532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:52.454798329Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T05:24:56.089919457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:24:56.545655959Z caller=checkpoint.go:378 msg="writing series" size="412 kB"
level=info ts=2026-04-22T05:24:57.65864207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="6.0 MB" avg_comp="2.0 MB" total_uncomp="50 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:24:57.6670901Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:25:00.635848546Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T05:25:01.090623339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:25:04.098614432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46568"
level=debug ts=2026-04-22T05:25:04.727050803Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=warn ts=2026-04-22T05:25:06.090366941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:08.818474559Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-22T05:25:08.818504804Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-22T05:25:10.623412879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:25:10.635198076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:25:11.090851046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:12.909285776Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=debug ts=2026-04-22T05:25:13.991557905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:25:14.709677636Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:25:16.090767008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:25:16.324784857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:25:16.999321944Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-22T05:25:21.090256626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:25:21.090271347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:21.090284182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:25:21.090293059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:25:21.090466809Z caller=checkpoint.go:378 msg="writing series" size="393 kB"
level=debug ts=2026-04-22T05:25:25.181430148Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T05:25:26.090247247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:25:28.49045756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:25:28.498062249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:25:29.27270412Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=debug ts=2026-04-22T05:25:29.272737623Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-22T05:25:31.08991593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:33.369329065Z caller=checkpoint.go:378 msg="writing series" size="9.8 MB"
level=debug ts=2026-04-22T05:25:33.36935429Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.8 MB" series=1
level=warn ts=2026-04-22T05:25:36.090141287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:37.454635565Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-22T05:25:41.089929734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:41.545101492Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=debug ts=2026-04-22T05:25:45.636020499Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=info ts=2026-04-22T05:25:45.796091232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:25:45.801441059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:25:45.801465771Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:25:45.80149463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:25:45.805135877Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:25:46.090517166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:25:46.32649724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:25:48.032070281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:25:49.727446813Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=debug ts=2026-04-22T05:25:51.067039897Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:25:51.068118095Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:25:51.09055156Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:25:51.090590304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:25:51.09060188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:25:51.090639682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:53.817842756Z caller=checkpoint.go:378 msg="writing series" size="271 kB"
level=warn ts=2026-04-22T05:25:56.09011549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:25:57.909238723Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-22T05:26:00.407017944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:26:00.414821644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T05:26:00.634269457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:53058"
level=warn ts=2026-04-22T05:26:01.090675035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:02.000096535Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T05:26:06.090662483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:06.090722596Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-04-22T05:26:07.245162871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:26:09.802356656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:26:09.809336039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:26:10.181342311Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=warn ts=2026-04-22T05:26:11.090878317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:26:11.492140699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:26:11.498106179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:26:12.995049357Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:26:14.272698443Z caller=checkpoint.go:378 msg="writing series" size="416 kB"
level=debug ts=2026-04-22T05:26:14.272724497Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=10
level=warn ts=2026-04-22T05:26:16.090319741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:26:16.328864543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:26:18.363859361Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=info ts=2026-04-22T05:26:21.090625252Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:26:21.090642733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:21.090665183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:26:21.090675877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:26:22.45585775Z caller=checkpoint.go:378 msg="writing series" size="3.1 MB"
level=debug ts=2026-04-22T05:26:22.455881723Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=2
level=warn ts=2026-04-22T05:26:26.090356066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:26.544864269Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=debug ts=2026-04-22T05:26:30.636439965Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T05:26:31.090175039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:34.72674084Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-22T05:26:36.090471855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:26:38.254719664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:26:38.261489568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:26:38.817829574Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T05:26:40.407650797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:26:41.090447926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:42.908612409Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=warn ts=2026-04-22T05:26:46.090355284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:26:46.331903756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:26:46.999929387Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-04-22T05:26:50.974681993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:26:50.986526105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:26:51.066638071Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:26:51.068186503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:26:51.090627512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:26:51.09062601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:51.090660584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:26:51.090689075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:26:51.090714521Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-04-22T05:26:55.181398593Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:26:56.090427718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:26:59.272768235Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-22T05:26:59.543685193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59566"
level=warn ts=2026-04-22T05:27:01.089901117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:03.363600917Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T05:27:04.106798953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:27:04.216995502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:27:04.223517536Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:27:06.090179742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:07.454538801Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=warn ts=2026-04-22T05:27:11.090785566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:11.544843656Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=debug ts=2026-04-22T05:27:11.544868952Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=12
level=debug ts=2026-04-22T05:27:11.901861506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:27:15.636140923Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=warn ts=2026-04-22T05:27:16.090599901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:27:16.334487926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:27:19.727366798Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-04-22T05:27:21.090075193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:27:21.090102119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:21.090118774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:27:21.090130189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:27:23.818745261Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-04-22T05:27:23.818771537Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=3
level=warn ts=2026-04-22T05:27:26.090867044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:27:26.549401988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:27:26.554374113Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:27:27.90833065Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-22T05:27:30.640415387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45500"
level=warn ts=2026-04-22T05:27:31.090477255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:31.999758895Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T05:27:34.215366258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:27:34.223315719Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:27:36.090939052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:36.09101159Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-04-22T05:27:40.181660396Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:27:41.090259393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:44.278625311Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-22T05:27:44.278653937Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=5
level=warn ts=2026-04-22T05:27:46.090450452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:27:46.337033984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:27:47.841928842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:27:48.363848879Z caller=checkpoint.go:378 msg="writing series" size="279 kB"
level=info ts=2026-04-22T05:27:51.067667254Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:27:51.067677371Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:27:51.090180874Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:27:51.090210847Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:27:51.090220316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:27:51.090214001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:52.454114862Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=warn ts=2026-04-22T05:27:56.090573773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:27:56.545243478Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-22T05:27:57.709236508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:27:59.745853384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:27:59.751979493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:28:00.636586548Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-22T05:28:01.090426911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:28:04.113851932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44890"
level=debug ts=2026-04-22T05:28:04.727771904Z caller=checkpoint.go:378 msg="writing series" size="784 kB"
level=debug ts=2026-04-22T05:28:04.727798218Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-22T05:28:06.090694542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:28:06.985155565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:28:06.989020091Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:28:08.818479584Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-22T05:28:11.090675814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:28:12.908406156Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=info ts=2026-04-22T05:28:14.60000934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:28:14.606862476Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:28:16.090485649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:28:16.33910743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:28:16.999823813Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:28:21.090373856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:28:21.090393378Z caller=checkpoint.go:390 msg="flushing series" totalSize="318 kB" series=3
level=debug ts=2026-04-22T05:28:21.090405719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:28:21.090415784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:28:21.090433886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:28:21.093273746Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-22T05:28:21.097457417Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007430313s
level=debug ts=2026-04-22T05:28:23.676355947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:28:23.676419755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:28:23.680707661Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:28:26.090173213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:28:31.090699503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:28:36.090809349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:28:39.852345921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:28:39.859970424Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:28:41.090418732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:28:46.09085286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:28:46.340736622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:28:51.067575846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:28:51.067580331Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:28:51.09012192Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:28:51.090137325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:28:51.090135836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:28:51.090159262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:28:51.090167398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:28:51.090255882Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-22T05:28:51.090467903Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-22T05:28:55.12129867Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:28:56.090021042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:28:56.496468549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:28:56.498085493Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T05:28:57.587125294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:28:57.59257478Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:28:57.592598966Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:28:57.592628138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:28:57.598108423Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:28:58.022354796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:28:58.022441084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:28:58.030395261Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=info ts=2026-04-22T05:28:58.198080268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:28:58.201817259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:28:59.150739475Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=warn ts=2026-04-22T05:29:01.090245076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:03.180861917Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-22T05:29:06.09030502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:07.210797561Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-04-22T05:29:09.178124349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:29:09.185579775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T05:29:10.281314392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:29:10.291323098Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:29:10.291354024Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:29:10.291390294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:29:10.298392951Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:29:11.090362944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:11.240802232Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=debug ts=2026-04-22T05:29:15.270114234Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-04-22T05:29:15.45953202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:29:15.466984132Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:29:16.090432598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:29:16.344027487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T05:29:17.25240129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:29:17.256576985Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:29:19.299967792Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-22T05:29:21.090122628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:29:21.090157486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:29:21.090152135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:29:21.090166009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:29:23.329875125Z caller=checkpoint.go:378 msg="writing series" size="479 kB"
level=debug ts=2026-04-22T05:29:24.22852688Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:29:25.278007292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:29:25.285441585Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:29:26.090598069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:27.359395123Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=debug ts=2026-04-22T05:29:27.359422455Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
ts=2026-04-22T05:29:29.557464462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51260"
level=warn ts=2026-04-22T05:29:31.090293912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:31.389127404Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
ts=2026-04-22T05:29:34.121015325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51252"
level=debug ts=2026-04-22T05:29:35.351911975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:29:35.4191259Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:29:36.090194458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:39.448915852Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-22T05:29:40.230006738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:29:40.239325072Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:29:41.090084921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:43.47912746Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-04-22T05:29:44.794726529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:29:44.800312762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:29:46.090252613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:29:46.347465262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:29:47.508812361Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T05:29:47.736512371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:29:47.741095203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:29:48.581123978Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:29:51.06732779Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:29:51.068399064Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:29:51.089786996Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:29:51.089820159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:29:51.089828649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:29:51.089850339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:29:51.539108037Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-22T05:29:55.568732338Z caller=checkpoint.go:378 msg="writing series" size="536 kB"
level=warn ts=2026-04-22T05:29:56.090799402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:29:59.560421813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52872"
level=debug ts=2026-04-22T05:29:59.598830065Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-22T05:30:00.676289947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:30:00.686888482Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:30:01.090158282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:30:03.627944315Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=debug ts=2026-04-22T05:30:03.628000489Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-04-22T05:30:04.122619442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47566"
level=warn ts=2026-04-22T05:30:06.090151006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:30:06.995678848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:30:07.001466425Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:30:07.658491719Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-22T05:30:09.673323656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:30:11.090446358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:30:11.688644228Z caller=checkpoint.go:378 msg="writing series" size="522 kB"
level=debug ts=2026-04-22T05:30:15.717549787Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-04-22T05:30:16.090008196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:30:16.349917036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:30:19.748356287Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=info ts=2026-04-22T05:30:21.090397953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:30:21.09042742Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:30:21.090435736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:30:21.090452472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:30:23.192156326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:30:23.202402346Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:30:23.542398238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:30:23.777971682Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-22T05:30:23.77800496Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=5
level=warn ts=2026-04-22T05:30:26.090355271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:30:27.754940247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:30:27.759083662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:30:27.807254777Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=warn ts=2026-04-22T05:30:31.089858872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:30:31.84567216Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-22T05:30:31.845696201Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=2
ts=2026-04-22T05:30:34.124305827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50704"
level=debug ts=2026-04-22T05:30:35.867248431Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-22T05:30:36.090722001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:30:38.682370129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:30:38.691287328Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:30:39.896765124Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=warn ts=2026-04-22T05:30:41.090622584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:30:43.927011316Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:30:46.090511886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:30:46.352344611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:30:47.956505082Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T05:30:51.067659298Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:30:51.067675838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=debug ts=2026-04-22T05:30:51.068996384Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T05:30:51.069042827Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974261 n=1
level=debug ts=2026-04-22T05:30:51.069094814Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974261 n=1
level=info ts=2026-04-22T05:30:51.090222244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:30:51.090251394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:30:51.090262361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:30:51.090269531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:30:51.987052359Z caller=checkpoint.go:378 msg="writing series" size="290 kB"
level=debug ts=2026-04-22T05:30:52.254924105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:30:55.995253049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="64 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:30:56.003092811Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:30:56.016698081Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-22T05:30:56.090001129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:30:57.413784807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:30:57.41855349Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:31:00.045891613Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-22T05:31:00.65433932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52938"
level=warn ts=2026-04-22T05:31:01.090728994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:04.076266471Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-22T05:31:06.090116268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:08.105710484Z caller=checkpoint.go:378 msg="writing series" size="513 B"
level=debug ts=2026-04-22T05:31:09.097143523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:31:11.09019309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:12.136049687Z caller=checkpoint.go:378 msg="writing series" size="572 kB"
level=debug ts=2026-04-22T05:31:12.136075933Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=10
level=info ts=2026-04-22T05:31:13.288293958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:31:13.295741867Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:31:16.090626168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:16.166058302Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
ts=2026-04-22T05:31:16.354599644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:31:18.093848446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:31:20.200468325Z caller=checkpoint.go:378 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-22T05:31:20.200493033Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.8 MB" series=2
level=info ts=2026-04-22T05:31:21.090203415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:31:21.090210438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:21.090240997Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:31:21.090252775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:31:22.459630632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:31:22.463517459Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:31:24.225229051Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:31:26.090874183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:28.255554495Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-04-22T05:31:29.407738175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:31:29.415430456Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-22T05:31:29.566392557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48980"
level=warn ts=2026-04-22T05:31:31.090658984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:32.28517689Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-22T05:31:34.129256338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57734"
level=warn ts=2026-04-22T05:31:36.090014122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:36.314914909Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=debug ts=2026-04-22T05:31:40.344480507Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T05:31:41.084200914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:31:41.089894681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-22T05:31:41.091493356Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:31:44.374506225Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-22T05:31:46.090037171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:31:46.356858304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:31:48.404520772Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=debug ts=2026-04-22T05:31:49.026274596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:31:49.027252127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:31:49.033723611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:31:49.924781821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:31:51.067035765Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:31:51.068101192Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:31:51.090604731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:31:51.090617829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:51.090641175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:31:51.090652095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:31:52.434416932Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-04-22T05:31:56.090581855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:31:56.463812464Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-22T05:32:00.493700394Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=debug ts=2026-04-22T05:32:00.493733668Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-22T05:32:01.090101613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:04.523300554Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T05:32:06.090467007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:08.55360999Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=info ts=2026-04-22T05:32:09.369901537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:32:09.379487378Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:32:09.379520034Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:32:09.379561218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:32:09.384222417Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:32:10.239900761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:32:10.247234145Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:32:11.090328563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:12.584003116Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=warn ts=2026-04-22T05:32:16.090730943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:32:16.358816344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:32:16.613938191Z caller=checkpoint.go:378 msg="writing series" size="283 kB"
level=debug ts=2026-04-22T05:32:20.643059638Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=debug ts=2026-04-22T05:32:20.643086018Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-22T05:32:21.09080628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:32:21.090829213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:21.090850716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:32:21.090859696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:32:21.683419658Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:32:23.983342838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:32:23.990072177Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:32:24.674335614Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-22T05:32:24.674364194Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=1
level=info ts=2026-04-22T05:32:25.414677828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:32:25.419338648Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:32:26.090640746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:32:28.560972089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:32:28.566138529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:32:28.702551327Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:32:31.090920419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:32.732898328Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-22T05:32:36.090764593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:36.762333756Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=info ts=2026-04-22T05:32:40.071840818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:32:40.080055584Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:32:40.792827906Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=debug ts=2026-04-22T05:32:40.880056013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:32:40.882593321Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T05:32:41.090877307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:44.836857563Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-22T05:32:44.836883114Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=5
level=warn ts=2026-04-22T05:32:46.090864952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:32:46.360935182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:32:48.852365169Z caller=checkpoint.go:378 msg="writing series" size="402 kB"
level=info ts=2026-04-22T05:32:49.385013446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:32:49.390027743Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:32:51.067439617Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:32:51.067450642Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:32:51.089937361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:32:51.089941152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:51.089982986Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:32:51.089992872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:32:52.881771867Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-22T05:32:52.882810765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:32:56.090034864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:32:56.912333964Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-22T05:33:00.665709425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57748"
level=debug ts=2026-04-22T05:33:00.941804225Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=warn ts=2026-04-22T05:33:01.09058818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:33:04.971880473Z caller=checkpoint.go:378 msg="writing series" size="758 kB"
level=debug ts=2026-04-22T05:33:04.971928504Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-04-22T05:33:06.090016573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:33:08.299154665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:33:08.307856705Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:33:09.001781088Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-22T05:33:11.090105623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:33:12.699718768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:33:12.704084275Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:33:13.030907815Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=info ts=2026-04-22T05:33:15.157937635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:33:15.163884324Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:33:16.089898284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:33:16.363426194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:33:17.060773098Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T05:33:17.893515566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:33:18.443340504Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:33:21.090142477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:33:21.090180635Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:33:21.09017478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:33:21.090194205Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:33:21.091257823Z caller=checkpoint.go:390 msg="flushing series" totalSize="338 kB" series=3
level=info ts=2026-04-22T05:33:21.095915787Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-22T05:33:21.096921899Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006525396s
level=warn ts=2026-04-22T05:33:26.090528956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:33:26.583408079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:33:26.592809139Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:33:31.09074411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:33:34.138460734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60542"
level=warn ts=2026-04-22T05:33:36.090078085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:33:41.089910657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:33:44.818077127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:33:46.007441096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:33:46.01232356Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:33:46.090781446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:33:46.365094186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:33:51.067340564Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:33:51.068412256Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:33:51.089806748Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:33:51.089804067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:33:51.089842152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:33:51.089854002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:33:51.089882187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:33:51.090172659Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-22T05:33:51.090472853Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-22T05:33:55.120633141Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:33:56.090150662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:33:58.049484022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:33:59.150208852Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-22T05:33:59.576153705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60586"
level=warn ts=2026-04-22T05:34:01.090621108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:34:02.495778769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:34:02.501291182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:34:03.180295612Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-22T05:34:04.27343263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:34:04.280619768Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:34:06.090904343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:07.210992046Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=info ts=2026-04-22T05:34:09.983837176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:34:09.988513577Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:34:11.090139623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:11.240510383Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-22T05:34:12.507173015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:34:12.511999264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-22T05:34:12.514080977Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:34:15.270300698Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-22T05:34:16.09052489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:34:16.367331549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:34:19.300322758Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-22T05:34:21.090725511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:34:21.090768816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:34:21.090781057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:34:21.090804861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:23.330022758Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=warn ts=2026-04-22T05:34:26.090032921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:27.359604693Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=debug ts=2026-04-22T05:34:27.359628694Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-04-22T05:34:31.090480311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:31.389821175Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=debug ts=2026-04-22T05:34:35.419374833Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:34:36.090010217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:34:38.315562483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="123 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:34:38.322175956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:34:39.449454464Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-22T05:34:41.014874528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:34:41.090472103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:34:43.450507211Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:34:43.478970516Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-22T05:34:46.089933292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:34:46.369919642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:34:47.509373606Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T05:34:51.067499606Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:34:51.067996978Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:34:51.090420183Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:34:51.090449232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:34:51.090438801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:34:51.090457869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:34:51.539042407Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=debug ts=2026-04-22T05:34:55.569131323Z caller=checkpoint.go:378 msg="writing series" size="607 kB"
level=warn ts=2026-04-22T05:34:56.090479954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:34:59.580472865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35386"
level=debug ts=2026-04-22T05:34:59.598082344Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-22T05:35:01.007827631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:35:01.015484651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:35:01.089881865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:35:03.607475215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:35:03.611814553Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:35:03.611835758Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:35:03.611862206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:35:03.615348329Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:35:03.628579745Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=debug ts=2026-04-22T05:35:03.628592931Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-22T05:35:04.18023897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=1 total_comp="3.1 MB" avg_comp="3.1 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:35:04.187607672Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:35:06.090219306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:07.658512136Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-04-22T05:35:07.878269469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:35:07.885216036Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:35:09.163325521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:35:11.09037774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:11.688115076Z caller=checkpoint.go:378 msg="writing series" size="668 kB"
level=debug ts=2026-04-22T05:35:15.717986338Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=warn ts=2026-04-22T05:35:16.090672485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:35:16.37326613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T05:35:19.207330778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="123 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:35:19.213971566Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:35:19.747516425Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=debug ts=2026-04-22T05:35:19.747540687Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-04-22T05:35:21.090192359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:35:21.090210191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:21.090222593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:35:21.0902334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:35:22.525303017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:35:22.529653033Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:35:23.777283891Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T05:35:26.09068516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:27.807371818Z caller=checkpoint.go:378 msg="writing series" size="420 kB"
level=warn ts=2026-04-22T05:35:31.089962731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:31.84720999Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-22T05:35:31.847235425Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=3
level=debug ts=2026-04-22T05:35:35.866980716Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-22T05:35:36.090169212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:36.227395289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:35:39.897143214Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T05:35:41.090746879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:35:41.789784895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:35:41.79606923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:35:41.796097812Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:35:41.796129807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:35:41.801630364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:35:42.547279209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:35:43.926284412Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:35:46.090064002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:35:46.332876307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:35:46.340067584Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:35:46.340091149Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:35:46.340117111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:35:46.346633512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
ts=2026-04-22T05:35:46.375990028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:35:47.956593065Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T05:35:48.581619163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:35:48.588347969Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:35:48.588367258Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:35:48.588392807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:35:48.594990875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:35:50.95508812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:35:50.95898606Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:35:51.066448042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:35:51.067521093Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:35:51.090053516Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:35:51.090062787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:35:51.090082711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:35:51.090103675Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:35:51.986371649Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=info ts=2026-04-22T05:35:55.083151446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:35:55.08906624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:35:56.016485813Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-22T05:35:56.090008317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:00.04663311Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T05:36:01.090785036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:04.076113812Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-04-22T05:36:04.150634305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33190"
level=info ts=2026-04-22T05:36:05.619556547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:36:05.629435762Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:36:05.720351645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:36:05.730887892Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:36:06.09046419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:08.105709367Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-22T05:36:11.090882297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:12.136288222Z caller=checkpoint.go:378 msg="writing series" size="880 kB"
level=debug ts=2026-04-22T05:36:12.136315507Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=10
level=info ts=2026-04-22T05:36:13.227661162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="123 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:36:13.231944679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:36:14.573205061Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:36:16.090516965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:16.165857889Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
ts=2026-04-22T05:36:16.378373256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:36:20.011497842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:36:20.013246051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:36:20.200709547Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-22T05:36:20.200733615Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.9 MB" series=2
level=info ts=2026-04-22T05:36:21.089915509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:36:21.089966471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T05:36:21.08994764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:36:21.089975862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:36:24.225733237Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:36:26.090221371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:28.255115715Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-04-22T05:36:29.588402711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53990"
level=warn ts=2026-04-22T05:36:31.09028984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:36:32.071609311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:36:32.078253342Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:36:32.284913551Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-22T05:36:33.572477011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:36:33.579037415Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T05:36:34.152250346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58852"
level=warn ts=2026-04-22T05:36:36.090794057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:36.314814134Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-04-22T05:36:38.168393017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.1 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:36:38.177167406Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:36:39.240366287Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:36:40.344329915Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:36:41.08986045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:36:41.46222472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:36:41.46613354Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:36:44.37499694Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:36:46.090783734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:36:46.380723255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:36:48.404734401Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=debug ts=2026-04-22T05:36:51.066805091Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:36:51.06787319Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:36:51.090352866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:36:51.090365739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:51.090382165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:36:51.090391928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:36:52.434277435Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-22T05:36:56.090040058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:36:56.464360114Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-22T05:36:58.359321418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="123 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:36:58.364062678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:37:00.493974871Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T05:37:01.090313322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:01.957023321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:37:04.524179768Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T05:37:06.090229653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:08.55379944Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=debug ts=2026-04-22T05:37:08.553823065Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-22T05:37:11.089926391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:12.584134209Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=debug ts=2026-04-22T05:37:13.146671435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:37:16.090496209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:37:16.383246344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:37:16.613143716Z caller=checkpoint.go:378 msg="writing series" size="405 kB"
level=debug ts=2026-04-22T05:37:20.642770409Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-22T05:37:20.648804752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:37:20.658096484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:37:21.090269412Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:37:21.090295523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:37:21.090306051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:37:21.090320665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:24.674284271Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-22T05:37:24.674313366Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=4
level=warn ts=2026-04-22T05:37:26.090525823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:37:27.122156762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:37:27.128041107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:37:28.703341032Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:37:31.090291649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:32.732302491Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=info ts=2026-04-22T05:37:32.741371546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:37:32.746677198Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T05:37:33.958219901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.1 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:37:33.965609174Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:37:36.089882181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:36.762797409Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=debug ts=2026-04-22T05:37:39.889137485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:37:40.792424424Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:37:41.090432186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:37:42.27760515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:37:42.28151944Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:37:42.351691827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:37:44.839899973Z caller=checkpoint.go:378 msg="writing series" size="18 MB"
level=debug ts=2026-04-22T05:37:44.839925862Z caller=checkpoint.go:390 msg="flushing series" totalSize="18 MB" series=5
level=warn ts=2026-04-22T05:37:46.09054356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:37:46.385007134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:37:48.852518343Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=info ts=2026-04-22T05:37:51.067517741Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:37:51.06751907Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:37:51.089988519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:37:51.090002511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:51.090018355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:37:51.090029316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:37:52.882280162Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=warn ts=2026-04-22T05:37:56.090213466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:37:56.912155569Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-22T05:38:00.942087908Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-04-22T05:38:01.090008866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:04.972682091Z caller=checkpoint.go:378 msg="writing series" size="992 kB"
level=debug ts=2026-04-22T05:38:04.972707811Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-04-22T05:38:05.04240588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:38:05.053786421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:38:06.090120518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:07.895146394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:38:09.001395527Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-22T05:38:11.090356388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:13.031337022Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=info ts=2026-04-22T05:38:15.236137829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:38:15.241677203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:38:16.090411274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:38:16.386688156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:38:17.061023276Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:38:21.090049291Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:38:21.090081356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:38:21.090091266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:38:21.090086611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:21.091241519Z caller=checkpoint.go:390 msg="flushing series" totalSize="368 kB" series=3
level=info ts=2026-04-22T05:38:21.094042877Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-22T05:38:21.09500803Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00464541s
level=warn ts=2026-04-22T05:38:26.090480179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:38:26.70663045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:38:26.713347742Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-22T05:38:30.690303266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60824"
level=warn ts=2026-04-22T05:38:31.089895544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:38:33.700382626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.1 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:38:33.707657037Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:38:34.169731349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:38:36.090087721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:38:37.818620177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:38:37.823733927Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:38:37.823756054Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:38:37.823787284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:38:37.827285694Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:38:41.090210714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:42.679926167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:38:46.090834879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:38:46.389498287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T05:38:49.575977477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:38:49.58254357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:38:51.067256995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:38:51.068333124Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:38:51.089698614Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:38:51.089809347Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-22T05:38:51.089913259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:38:51.089931878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:38:51.089942155Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:38:51.089934597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:51.090020681Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-22T05:38:55.120420987Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:38:56.090647674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:38:59.150280971Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-22T05:39:00.693168681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49338"
level=warn ts=2026-04-22T05:39:01.090154617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:03.179710628Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-22T05:39:03.428292522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:39:03.438504483Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T05:39:04.166336166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42610"
level=info ts=2026-04-22T05:39:04.686327046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:39:04.69301946Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:39:06.090170761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:07.210685807Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=debug ts=2026-04-22T05:39:08.662561906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:39:11.090026028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:11.239400234Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-22T05:39:14.92147339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:39:14.928913425Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:39:15.269626177Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-22T05:39:16.090385504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:39:16.391638928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:39:19.299897274Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-22T05:39:19.838699222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:39:19.843099225Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:39:21.090202244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:39:21.090235733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:39:21.090249586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:39:21.090267977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:23.33008057Z caller=checkpoint.go:378 msg="writing series" size="505 kB"
level=debug ts=2026-04-22T05:39:23.330115917Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-22T05:39:26.090616455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:27.359799137Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=debug ts=2026-04-22T05:39:29.594874732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:39:30.00496503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=2 total_comp="6.1 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:39:30.013861421Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:39:31.090511362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:31.38946497Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
ts=2026-04-22T05:39:34.167776922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54142"
level=debug ts=2026-04-22T05:39:35.419118979Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:39:36.090270171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:39.448598492Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-22T05:39:41.090191103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:41.440237493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:39:43.47870736Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=info ts=2026-04-22T05:39:44.839271477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:39:44.849938967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:39:46.090027119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:39:46.393067158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:39:47.508747485Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T05:39:51.066978921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:39:51.067990035Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:39:51.090390541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:39:51.090424105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:39:51.090435597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:39:51.090444512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:51.538311407Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=debug ts=2026-04-22T05:39:55.568859093Z caller=checkpoint.go:378 msg="writing series" size="704 kB"
level=debug ts=2026-04-22T05:39:55.568882769Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-04-22T05:39:55.996735953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:39:56.002534646Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:39:56.090040866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:39:59.59786235Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-22T05:40:01.090654449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:40:01.280298035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:40:01.288007032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:40:01.88453288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:40:01.898366574Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:40:03.627848611Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-22T05:40:06.090363862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:07.658080134Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T05:40:11.089867972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:11.688013419Z caller=checkpoint.go:378 msg="writing series" size="892 kB"
level=debug ts=2026-04-22T05:40:11.688038044Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-04-22T05:40:15.396568494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:40:15.718087147Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-04-22T05:40:16.090032633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:40:16.394243264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:40:19.747728424Z caller=checkpoint.go:378 msg="writing series" size="486 kB"
level=warn ts=2026-04-22T05:40:21.09030979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:40:21.090308147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:40:21.090351564Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:40:21.090384452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:40:23.777068531Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T05:40:26.089941576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:27.157428345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:40:27.159201743Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T05:40:27.80705504Z caller=checkpoint.go:378 msg="writing series" size="419 kB"
level=info ts=2026-04-22T05:40:29.019346675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:40:29.028233327Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
ts=2026-04-22T05:40:29.60734764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36198"
level=info ts=2026-04-22T05:40:30.5649383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:40:30.570174868Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-22T05:40:30.699403979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:55086"
level=warn ts=2026-04-22T05:40:31.090000003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:31.849200892Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-22T05:40:31.849224589Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=5
level=debug ts=2026-04-22T05:40:35.866885598Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-22T05:40:36.090889181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:39.896450427Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-04-22T05:40:40.961239967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:40:40.9682628Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:40:41.090543891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:43.926652702Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-22T05:40:46.09073719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:40:46.395628392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:40:47.955729578Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=debug ts=2026-04-22T05:40:48.009902337Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:40:49.866465604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:40:49.877481041Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:40:51.06675758Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:40:51.067805009Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:40:51.090228634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:40:51.090267361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:40:51.09027737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:40:51.090270277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:40:51.986273583Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=debug ts=2026-04-22T05:40:56.016284562Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-22T05:40:56.090925174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:40:57.584878071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:40:57.590027623Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:41:00.045276014Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-22T05:41:00.702662414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:56582"
level=warn ts=2026-04-22T05:41:01.090179231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:04.075295699Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
ts=2026-04-22T05:41:04.175812967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54860"
level=warn ts=2026-04-22T05:41:06.090110408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:08.105779953Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-22T05:41:11.090786072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:12.13598186Z caller=checkpoint.go:378 msg="writing series" size="877 kB"
level=debug ts=2026-04-22T05:41:12.136012932Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=10
level=info ts=2026-04-22T05:41:13.25650121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:41:13.266230679Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:41:16.090931397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:16.165459552Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
ts=2026-04-22T05:41:16.398269033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:41:19.295531233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:41:19.301925336Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:41:20.206262164Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-22T05:41:20.206288555Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-04-22T05:41:21.090299322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:41:21.090337908Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:41:21.090350497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:41:21.09042955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:41:21.101529607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:41:21.107306622Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:41:21.1073292Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:41:21.107356896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:41:21.110738272Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:41:21.98539745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:41:23.826999478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:41:24.225388129Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T05:41:26.090560554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:28.254343329Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-04-22T05:41:29.61145197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43670"
level=warn ts=2026-04-22T05:41:31.090120275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:32.284519866Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-22T05:41:34.17709539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49546"
level=warn ts=2026-04-22T05:41:36.089851768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:36.314691555Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-22T05:41:38.297746585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:41:38.302715034Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T05:41:39.575842538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:41:39.579181189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:41:39.582809344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:41:39.582824875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:41:40.344672256Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:41:41.090925794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:44.37438744Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=warn ts=2026-04-22T05:41:46.090289928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:41:46.401057964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:41:47.362779499Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:41:48.403771738Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-22T05:41:51.067734511Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:41:51.068362509Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:41:51.090648306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:41:51.090677972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:41:51.090686823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:41:51.090645786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:52.433573579Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=warn ts=2026-04-22T05:41:56.090208032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:41:56.464039804Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-22T05:41:57.814155866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:41:59.348500158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:41:59.354115192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:41:59.354136404Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:41:59.35416158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:41:59.358659685Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:42:00.493634779Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T05:42:01.089865305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:04.523043713Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T05:42:06.090428809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:08.553730895Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=debug ts=2026-04-22T05:42:08.553757137Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-04-22T05:42:08.823554739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:42:08.828004586Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T05:42:10.39647868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:42:10.402150018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T05:42:10.582755224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:42:10.590240271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:42:11.090773095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:12.583400295Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=warn ts=2026-04-22T05:42:16.090134863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:42:16.403087499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:42:16.612694772Z caller=checkpoint.go:378 msg="writing series" size="505 kB"
level=debug ts=2026-04-22T05:42:19.788941982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:42:19.789036062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="908 kB" avg_comp="908 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:42:19.793999744Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:42:20.642583086Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=info ts=2026-04-22T05:42:21.090755336Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:42:21.090779131Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:42:21.09078657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:42:21.090794296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:24.675230858Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-22T05:42:24.675255623Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=4
level=warn ts=2026-04-22T05:42:26.090877598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:28.702879098Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-22T05:42:29.617163084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51236"
level=warn ts=2026-04-22T05:42:31.0902851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:32.732112895Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
ts=2026-04-22T05:42:34.181144327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38428"
level=info ts=2026-04-22T05:42:35.199161343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:42:35.207358227Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-22T05:42:36.090610779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:36.76197159Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=info ts=2026-04-22T05:42:37.990220325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:42:37.994458354Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:42:39.573897646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:42:39.580375515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:42:39.580398728Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:42:39.580426321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:42:39.586518363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:42:40.792544987Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:42:41.090554052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:41.624273341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:42:43.058165774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:42:43.063598627Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:42:44.711150362Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:42:44.853210675Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-04-22T05:42:44.85323754Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=5
level=warn ts=2026-04-22T05:42:46.090610913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:42:46.405518886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:42:48.852447203Z caller=checkpoint.go:378 msg="writing series" size="447 kB"
level=info ts=2026-04-22T05:42:50.058743293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:42:50.066174291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:42:51.067116596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:42:51.068180844Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:42:51.090476249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:42:51.090500353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:42:51.090508933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:42:51.090521726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:52.881755388Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=warn ts=2026-04-22T05:42:56.090700706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:42:56.91160561Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-22T05:42:57.41192464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:42:57.41768085Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:43:00.941223342Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-04-22T05:43:01.090364707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:03.460715369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:43:03.465977351Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:43:04.971644686Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-22T05:43:04.971669785Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
level=warn ts=2026-04-22T05:43:06.090198513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:07.222451504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:43:07.230256185Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:43:07.230278384Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:43:07.230305061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:43:07.236974667Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:43:08.758408309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:43:09.001058925Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=warn ts=2026-04-22T05:43:11.090982906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:43:13.030990858Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-22T05:43:16.090523023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:43:16.407423482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:43:17.060889133Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:43:18.437300463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:43:18.446472922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T05:43:21.027797617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:43:21.037310067Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=info ts=2026-04-22T05:43:21.090821275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:43:21.090839471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:43:21.090834445Z caller=checkpoint.go:390 msg="flushing series" totalSize="402 kB" series=3
level=debug ts=2026-04-22T05:43:21.090852776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:43:21.090862459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:43:21.096451707Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-22T05:43:21.097727385Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007788219s
level=debug ts=2026-04-22T05:43:21.189078182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T05:43:26.090943103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:43:31.090529391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:32.519258951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:43:32.526194957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:43:35.045807465Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:43:36.089890941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:39.325587917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:43:39.332071821Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:43:41.090879211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:41.408998143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:43:41.413490802Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T05:43:44.88551153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:43:44.893678722Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:43:46.09009095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:43:46.409159841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:43:51.066659499Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:43:51.067724069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:43:51.090069332Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:43:51.090106132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:43:51.090087977Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T05:43:51.090117736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:43:51.090134705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:43:51.09026344Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-22T05:43:51.0905343Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-04-22T05:43:52.372516518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:43:52.377088307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:43:55.121164242Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:43:56.090426948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:43:59.151220745Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-22T05:44:00.715387039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:42288"
level=warn ts=2026-04-22T05:44:01.090497015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:03.180268163Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-22T05:44:06.090690117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:07.211165748Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=debug ts=2026-04-22T05:44:08.58714305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:44:11.090339306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:11.240676361Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-22T05:44:11.274474636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:44:11.27594716Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T05:44:12.776236201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:44:12.787916749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:44:15.269863361Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=warn ts=2026-04-22T05:44:16.090746171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:44:16.410871757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:44:19.29977791Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-22T05:44:21.090154873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:44:21.090181926Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:44:21.090190958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:44:21.090191982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:23.330392541Z caller=checkpoint.go:378 msg="writing series" size="466 kB"
level=info ts=2026-04-22T05:44:25.019426229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:44:25.029247768Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:44:26.090230751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:44:26.949460724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:44:26.953494225Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:44:27.360035477Z caller=checkpoint.go:378 msg="writing series" size="261 kB"
level=debug ts=2026-04-22T05:44:27.360062431Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
ts=2026-04-22T05:44:29.624236074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47950"
level=warn ts=2026-04-22T05:44:31.09089615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:31.389608518Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T05:44:34.775554492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:44:34.780423163Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:44:35.419204267Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-22T05:44:36.090122737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:38.56457048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T05:44:39.22242732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:44:39.228105861Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:44:39.44977827Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=warn ts=2026-04-22T05:44:41.090940417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:43.479520171Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-22T05:44:46.090764381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:44:46.413690059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:44:46.733918752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:44:46.738819134Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:44:47.509095656Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T05:44:51.067045675Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:44:51.068210464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-22T05:44:51.090544652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:44:51.090565223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:44:51.090591403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:44:51.090606479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:44:51.538358751Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-22T05:44:55.569520207Z caller=checkpoint.go:378 msg="writing series" size="810 kB"
level=debug ts=2026-04-22T05:44:55.569543755Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-04-22T05:44:56.090701055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:44:59.598578157Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-22T05:45:01.09054303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:03.628502817Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
ts=2026-04-22T05:45:04.194013835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38598"
level=warn ts=2026-04-22T05:45:06.089929927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:07.55010162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:45:07.658536811Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-04-22T05:45:10.413247257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:45:10.420419081Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:45:11.090561642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:11.688333441Z caller=checkpoint.go:378 msg="writing series" size="938 kB"
level=debug ts=2026-04-22T05:45:11.688362718Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=debug ts=2026-04-22T05:45:14.188777554Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:45:15.718049271Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-04-22T05:45:16.090416037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:45:16.41651615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=info ts=2026-04-22T05:45:18.166252922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:45:18.171208153Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:45:19.747814447Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=info ts=2026-04-22T05:45:20.106007872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:45:20.112272502Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T05:45:21.090311391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:45:21.090354574Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:45:21.090376812Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:45:21.090417261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:45:21.696784633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:45:21.705588766Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:45:23.777500075Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-22T05:45:26.009965617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:45:26.014105303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:45:26.014126291Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:45:26.014158487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:45:26.017241256Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T05:45:26.09073089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:27.807936761Z caller=checkpoint.go:378 msg="writing series" size="548 kB"
level=debug ts=2026-04-22T05:45:27.80799761Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-22T05:45:31.090801022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:31.850929726Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-22T05:45:31.850970079Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=debug ts=2026-04-22T05:45:35.867558351Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-22T05:45:36.090481084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:39.897061669Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-22T05:45:41.090221827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:45:42.491519079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:45:42.497559489Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:45:42.821307969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:45:42.82611182Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:45:43.046337182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:45:43.927088991Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=debug ts=2026-04-22T05:45:44.016490053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:45:46.090801772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:45:46.418272003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:45:47.956283107Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=debug ts=2026-04-22T05:45:51.066630323Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:45:51.067669185Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:45:51.068939278Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T05:45:51.069004402Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974262 n=1
level=debug ts=2026-04-22T05:45:51.069055811Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974262 n=1
level=info ts=2026-04-22T05:45:51.090115232Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:45:51.090165645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:45:51.090176453Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:45:51.09019768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:45:51.987206251Z caller=checkpoint.go:378 msg="writing series" size="478 kB"
level=info ts=2026-04-22T05:45:54.366919154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:45:54.375345057Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:45:56.01613895Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-22T05:45:56.016163105Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T05:45:56.090940504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:00.047457732Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-04-22T05:46:00.724562087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34650"
level=warn ts=2026-04-22T05:46:01.090290233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:04.075783839Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-22T05:46:04.369556281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:46:04.374519188Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:46:05.078685301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:46:05.08323766Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-22T05:46:06.090191671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:08.105745577Z caller=checkpoint.go:378 msg="writing series" size="654 B"
level=info ts=2026-04-22T05:46:08.678496502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:46:08.685892421Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T05:46:11.090061353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:12.136474908Z caller=checkpoint.go:378 msg="writing series" size="880 kB"
level=info ts=2026-04-22T05:46:15.864884418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:46:15.87391508Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:46:16.090231872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:16.165663231Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=debug ts=2026-04-22T05:46:16.165685201Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-04-22T05:46:16.420353018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:46:19.522851457Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:46:20.205023566Z caller=checkpoint.go:378 msg="writing series" size="14 MB"
level=debug ts=2026-04-22T05:46:20.20506565Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-22T05:46:21.090786433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:46:21.090796144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:21.090814681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:46:21.090823244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:46:24.225485795Z caller=checkpoint.go:378 msg="writing series" size="9.3 kB"
level=warn ts=2026-04-22T05:46:26.090545454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:46:27.647065127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:46:27.654525867Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:46:28.255429302Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-04-22T05:46:29.632116572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:32906"
level=warn ts=2026-04-22T05:46:31.089928242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:32.284619291Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T05:46:36.090476491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:36.315433065Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=debug ts=2026-04-22T05:46:40.345191181Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T05:46:41.090009709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:46:42.373521283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:46:42.383876652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:46:43.482247757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:46:44.375027251Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=info ts=2026-04-22T05:46:44.507450441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:46:44.511452984Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:46:45.920736651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:46:46.090249098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:46:46.423105563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:46:48.404086762Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=debug ts=2026-04-22T05:46:51.066641378Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:46:51.068279428Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:46:51.090620468Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:46:51.090626547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:51.090686082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:46:51.090697952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:46:52.434144073Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-22T05:46:56.09087111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:46:56.464189594Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-22T05:46:59.635353891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38744"
level=debug ts=2026-04-22T05:47:00.493729857Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-22T05:47:00.730641762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:51256"
level=warn ts=2026-04-22T05:47:01.09055068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:47:01.52725021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:47:01.533796975Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T05:47:03.412361954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:47:03.416963611Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:47:04.524183436Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-22T05:47:05.37165162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:47:05.381859202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:47:06.090545498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:08.554352264Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=debug ts=2026-04-22T05:47:08.554376177Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-04-22T05:47:11.090433136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:12.584176879Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=debug ts=2026-04-22T05:47:13.321031952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:47:16.090660702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:47:16.424744714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8238fc8e 10.129.0.68:7946"
level=debug ts=2026-04-22T05:47:16.613156369Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=debug ts=2026-04-22T05:47:20.64303265Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-22T05:47:21.0907401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:47:21.090774649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:47:21.090785382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:47:21.09080039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:22.418616486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T05:47:24.676213224Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-22T05:47:24.676239533Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=4
level=info ts=2026-04-22T05:47:25.841225513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:47:25.845843406Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-22T05:47:26.090465211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:47:27.31877675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.6 MB" total_uncomp="171 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:47:27.322479995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:47:28.703331439Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T05:47:30.517656091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:47:30.524189694Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T05:47:30.733635933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45552"
level=warn ts=2026-04-22T05:47:31.090802859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:32.732669518Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-04-22T05:47:36.090686223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:36.762822462Z caller=checkpoint.go:378 msg="writing series" size="197 kB"
level=debug ts=2026-04-22T05:47:39.820079774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:47:40.793031159Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T05:47:41.090440266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:44.842718562Z caller=checkpoint.go:378 msg="writing series" size="24 MB"
level=debug ts=2026-04-22T05:47:44.84274432Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=warn ts=2026-04-22T05:47:46.090729504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:47:46.427000137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d65hp-2d364fc6 10.128.0.72:7946"
level=debug ts=2026-04-22T05:47:48.852634507Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=info ts=2026-04-22T05:47:51.067770583Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T05:47:51.068251173Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:47:51.090524669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T05:47:51.090534035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:51.090541198Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:47:51.090551882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T05:47:52.882731005Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-22T05:47:56.090679089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:47:56.911624866Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-22T05:47:57.270002324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:47:57.277200756Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:47:59.360998668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T05:47:59.362521142Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T05:48:00.734784789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-tcca\"}"
ts=2026-04-22T05:48:00.73621592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:51016"
level=error ts=2026-04-22T05:48:00.741850725Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-tcca\"}"
level=debug ts=2026-04-22T05:48:00.941581404Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
level=warn ts=2026-04-22T05:48:01.090526871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:48:03.345920889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:48:03.350729383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:48:03.350749474Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:48:03.350773147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:48:03.35452939Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-22T05:48:04.209657011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54414"
level=debug ts=2026-04-22T05:48:04.972382444Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-22T05:48:04.972409173Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=5
level=warn ts=2026-04-22T05:48:06.090090297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:48:09.001882017Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=warn ts=2026-04-22T05:48:11.09037468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:48:12.328311534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:48:12.333662658Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T05:48:12.849078939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T05:48:13.031964008Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-04-22T05:48:14.05991567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="196 MB" avg_uncomp="22 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:48:14.063968614Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T05:48:15.293115722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:48:15.297691144Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T05:48:15.637623442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:48:15.644822511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T05:48:16.090565022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:48:16.430406268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=debug ts=2026-04-22T05:48:17.060727597Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T05:48:21.089792372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:48:21.089830241Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:48:21.089843573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T05:48:21.089962743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:48:21.090891487Z caller=checkpoint.go:390 msg="flushing series" totalSize="421 kB" series=3
level=info ts=2026-04-22T05:48:21.09581096Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-22T05:48:21.097054822Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006605651s
level=warn ts=2026-04-22T05:48:26.090694499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:48:29.642846149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59338"
level=warn ts=2026-04-22T05:48:31.090279532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:48:36.090673204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:48:40.061524125Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:48:41.090835356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T05:48:46.09013391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T05:48:46.432801487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8tvbc-d4f937db 10.128.0.71:7946"
level=info ts=2026-04-22T05:48:50.198404845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T05:48:50.202281347Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db384e6d0%3A19db3a89093%3Afbc2f896?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-22T05:48:51.066997596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.94:11211 count=1
level=info ts=2026-04-22T05:48:51.068045417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T05:48:51.090342896Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T05:48:51.090401999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:48:51.090378228Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T05:48:51.090433627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T05:48:51.090449347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T05:48:51.090726203Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=debug ts=2026-04-22T05:48:51.090989961Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=debug ts=2026-04-22T05:48:51.153288901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T05:48:53.399675187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:48:53.404046637Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:48:53.4040691Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:48:53.404092371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:48:53.407816259Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-22T05:48:54.152003882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:48:54.157150297Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:48:54.157170393Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:48:54.157196108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.0 MB" avg_uncomp="8.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T05:48:54.161662147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db384ecba%3A19db39c9392%3A1df63eb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T05:48:55.120835394Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T05:48:56.09077144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:48:59.150762134Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=info ts=2026-04-22T05:49:00.414168875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=00bfd940976618a0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-tcca\"}"
level=error ts=2026-04-22T05:49:00.421893603Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bfd940976618a0/19db39cbe89%3A19db3a987fb%3Ac5b70c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-tcca\"}"
level=warn ts=2026-04-22T05:49:01.090572211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:49:03.181568691Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
ts=2026-04-22T05:49:04.213223575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44418"
level=info ts=2026-04-22T05:49:04.375011412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="196 MB" avg_uncomp="22 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T05:49:04.379035065Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db385578a%3A19db388445b%3A26ae5248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T05:49:04.415188144Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T05:49:06.090155917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T05:49:06.798783812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:49:06.807140442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:49:06.807169953Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T05:49:06.807210475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T05:49:06.815274015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db384e7a3%3A19db3854dfc%3A1eb6d0bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T05:49:07.210582445Z caller=checkpoint.go:378 msg="writing series" size="278 kB"
level=info ts=2026-04-22T05:49:10.716264792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="909 kB" avg_comp="909 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T05:49:10.721168062Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db3850ba5%3A19db3b4f31e%3Afa75cf1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-22T05:49:11.090617816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T05:49:11.24110931Z caller=checkpoint.go:378 msg="writing series" size="49 kB"