level=info ts=2026-05-01T22:45:57.443871234Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-01T22:45:57.44390947Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-01T22:45:57.44396568Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-01T22:45:57.445400483Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-01T22:45:57.446374867Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T22:45:57.446469207Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.87 inf=eth0
level=info ts=2026-05-01T22:45:57.447210637Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-8fc2a50b
ts=2026-05-01T22:45:57.447290689Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-01T22:45:57.447308335Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.87 advertisePort=7946
level=debug ts=2026-05-01T22:45:57.447335357Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.87 advertisePort=7946
level=info ts=2026-05-01T22:45:57.44749438Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-01T22:45:57.452672663Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T22:45:57.452694217Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:45:57.452710845Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.218398ms
level=error ts=2026-05-01T22:45:57.452723268Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-01T22:45:57.452739331Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-05-01T22:45:57.455624389Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T22:45:57.455678829Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-01T22:45:57.456491003Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:45:57.456520105Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-01T22:45:57.457314807Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-01T22:45:57.457484826Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:45:57.458028904Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-01T22:45:57.458096406Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-01T22:45:57.458048539Z
level=debug ts=2026-05-01T22:45:57.458155373Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-01T22:45:57.458275997Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-01T22:45:57.45830886Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-01T22:45:57.458317699Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-01T22:45:57.458316138Z
level=debug ts=2026-05-01T22:45:57.45832455Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-01T22:45:57.460359545Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T22:45:57.460444852Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.87 inf=eth0
level=info ts=2026-05-01T22:45:57.474626904Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-01T22:45:57.474688553Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-01T22:45:57.474692955Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-01T22:45:57.474709558Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-01T22:45:57.474724763Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-01T22:45:57.474732776Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-01T22:45:57.474739999Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-01T22:45:57.474749274Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-01T22:45:57.474753481Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-01T22:45:57.474777304Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-01T22:45:57.474798863Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-01T22:45:57.474903434Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-01T22:45:57.474911126Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-01T22:45:57.474912415Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-01T22:45:57.474932645Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-01T22:45:57.475006689Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-01T22:45:57.47504652Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-01T22:45:57.475052832Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-01T22:45:57.475057836Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-01T22:45:57.475154232Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-01T22:45:57.475159467Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-01T22:45:57.475162847Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-01T22:45:57.475165747Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-01T22:45:57.475168581Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-01T22:45:57.475171125Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-01T22:45:57.475189027Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-01T22:45:57.475196036Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T22:45:57.475233578Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.966µs success=true
level=info ts=2026-05-01T22:45:57.475245866Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-01T22:45:57.475374059Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=186.868µs errors=false
level=info ts=2026-05-01T22:45:57.475389625Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-01T22:45:57.475873028Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=682.981µs errors=false
level=info ts=2026-05-01T22:45:57.475889928Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-01T22:45:57.47589771Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=710.524µs
level=info ts=2026-05-01T22:45:57.475916094Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-01T22:45:57.47600303Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-01T22:45:57.476017688Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.936348802s
level=info ts=2026-05-01T22:45:57.476027675Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-01T22:45:57.476101898Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-05-01T22:45:57.476142389Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-01T22:45:57.476150506Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-01T22:45:57.476207637Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-01T22:45:57.476163557Z caller=loki.go:615 msg="Loki started" startup_time=114.459372ms
ts=2026-05-01T22:45:58.506550755Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:45:58.506581081Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:46:01.727905267Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:46:01.727928114Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:46:02.476179956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:46:06.076150685Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:46:06.076172622Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:46:07.476284609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:46:12.477020824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:46:17.325728292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
level=info ts=2026-05-01T22:46:17.326525285Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=19.873778796s
level=warn ts=2026-05-01T22:46:17.47692815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:46:21.054222635Z caller=logging.go:137 msg="GET /ready (503) 119.019µs"
level=warn ts=2026-05-01T22:46:22.477262633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:46:27.47679312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-01T22:46:27.476822645Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-01T22:46:27.476833445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:46:27.47686948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:46:31.053377749Z caller=logging.go:137 msg="GET /ready (503) 66.404µs"
level=warn ts=2026-05-01T22:46:32.477359434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:46:33.210295669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:46:33.397785662Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:46:33.397839223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:46:33.405545321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:46:37.476730996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:46:42.477202798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:46:47.476496578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:46:50.760686351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34480"
level=warn ts=2026-05-01T22:46:52.477120898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:46:52.820329041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-01T22:46:53.337903548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:46:53.344011062Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T22:46:54.383154169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43610"
level=debug ts=2026-05-01T22:46:57.45697874Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:46:57.457968071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:46:57.476401373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:46:57.476419169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:46:57.476429024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:46:57.476445387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:47:02.47635207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:47:03.212164409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:47:05.89317965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:47:07.47698268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:47:12.4771292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:47:12.574404983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:47:17.476975423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:47:22.476799802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:47:27.47626246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:47:27.476291387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:47:27.476302547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:47:27.476286933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:47:32.476432821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:47:32.479491551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:47:32.489691154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T22:47:33.213661714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:47:34.730676654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:47:37.476507754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:47:38.346709386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:47:42.476498232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:47:47.476764011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:47:50.763197848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34884"
level=warn ts=2026-05-01T22:47:52.476417822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:47:54.387540289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40976"
level=debug ts=2026-05-01T22:47:56.066270203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T22:47:57.456930855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:47:57.457962071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:47:57.476404029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:47:57.476437534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:47:57.476452169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:47:57.476458838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:02.476462493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:48:03.215223417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=warn ts=2026-05-01T22:48:07.476203037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:48:11.00740019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:48:12.477043465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:17.476555651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:22.476694411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:48:24.389663885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57162"
level=info ts=2026-05-01T22:48:27.47626601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:48:27.476280605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:48:27.476299685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:48:27.476314955Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:48:30.682313359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T22:48:30.68420506Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T22:48:32.424175077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:48:32.432004939Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:48:32.476919573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:48:33.216564617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:48:35.13388409Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:48:37.476787449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:42.477152771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:47.477053842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:48:52.476740342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:48:54.392032252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43214"
level=info ts=2026-05-01T22:48:57.457617959Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T22:48:57.457880193Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:48:57.476241366Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:48:57.476259558Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:48:57.476280327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:48:57.47629002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:48:59.910489492Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:49:02.477209675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:49:03.218695034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=warn ts=2026-05-01T22:49:07.476806002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:49:12.4769279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:49:16.523520204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:49:17.477001064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:49:22.477154424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:49:27.476159027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:49:27.476191593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:49:27.476210515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:49:27.476220276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:49:30.307421092Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:49:30.926160087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:49:30.934583291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:49:32.476464924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:49:33.220575748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=warn ts=2026-05-01T22:49:37.476362406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:49:42.476911022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:49:47.476758483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:49:52.476977013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:49:57.457392677Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:49:57.458519955Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:49:57.476855063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:49:57.476874103Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T22:49:57.476876121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:49:57.476882254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:50:01.434548991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:50:02.477128585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:50:03.221752381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=warn ts=2026-05-01T22:50:07.476801828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:50:12.477127056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:50:13.920973917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:50:17.476473786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:50:20.771557023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40862"
level=warn ts=2026-05-01T22:50:22.477207239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:50:24.398548721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58640"
level=info ts=2026-05-01T22:50:26.511818879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:50:26.520190843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T22:50:27.476132668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:50:27.476145299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:50:27.476159542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:50:27.476169619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:50:27.593583503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:50:32.477132279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:50:33.223331375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=warn ts=2026-05-01T22:50:37.476807975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:50:42.47666112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:50:47.476257616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:50:52.4765157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:50:57.011220127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:50:57.457048739Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:50:57.458097448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T22:50:57.476151358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:50:57.476152495Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-01T22:50:57.476166379Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-01T22:50:57.476173216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:50:57.476211569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T22:50:57.476295478Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-01T22:50:57.476303273Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T22:50:57.476317796Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.951µs success=true
level=debug ts=2026-05-01T22:50:57.476481998Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-01T22:51:01.831436717Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-01T22:51:02.477142935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:51:03.224967375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T22:51:06.186601484Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-01T22:51:07.476221172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:51:08.14155372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:51:08.149012594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:51:09.330260687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T22:51:10.541902211Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=warn ts=2026-05-01T22:51:12.4763078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:14.896094846Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-01T22:51:17.476609821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:19.25147664Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-01T22:51:22.092689986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:51:22.477243901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:23.606261199Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-01T22:51:27.476338222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:51:27.476365581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T22:51:27.476374128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:51:27.476383234Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:51:27.96076105Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=debug ts=2026-05-01T22:51:32.316002183Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T22:51:32.476821569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:51:33.226325054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T22:51:36.670083009Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-01T22:51:37.476620738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:41.025906882Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-01T22:51:42.477054254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:45.380796142Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-01T22:51:47.476988431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:49.735496859Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-01T22:51:51.669332678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:51:52.476337288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:54.089741721Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-01T22:51:54.820999166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:51:54.828232075Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:51:57.456905296Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:51:57.457935655Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:51:57.476436897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:51:57.47645797Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:51:57.476465461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:51:57.476478301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:51:58.44503542Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T22:52:02.476646346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:02.800112091Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=debug ts=2026-05-01T22:52:02.800137916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=16
ts=2026-05-01T22:52:03.227422638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:52:05.872025315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T22:52:07.15438439Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-01T22:52:07.476951043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:11.509077664Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-01T22:52:12.476923453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:15.86381452Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T22:52:16.393103205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:52:17.476545248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:20.218766797Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-01T22:52:22.476331682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:52:24.405880609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34396"
level=debug ts=2026-05-01T22:52:24.573577991Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-05-01T22:52:27.476110351Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:52:27.476142824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T22:52:27.476132904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:52:27.47615136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:52:28.928966748Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T22:52:32.476790486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:52:33.228414379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:52:33.283891517Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
ts=2026-05-01T22:52:34.98726961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57780"
level=warn ts=2026-05-01T22:52:37.476409381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:37.63869362Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-01T22:52:41.993242466Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T22:52:42.476755093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:45.313983854Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:52:45.529697005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:52:45.540119464Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:52:46.348065155Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T22:52:47.476693678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:50.702658023Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T22:52:52.47645498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:52:54.408342929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51984"
level=debug ts=2026-05-01T22:52:55.058042471Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T22:52:57.457561127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T22:52:57.457831121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:52:57.476215498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:52:57.476234511Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:52:57.476242834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:52:57.476245087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:52:59.412732486Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-01T22:53:02.476494772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:53:03.229660584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T22:53:03.766856853Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-01T22:53:07.476506456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:08.122491756Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-01T22:53:12.476445605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:12.476483437Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-01T22:53:14.921716314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:53:14.92177144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:53:14.925357161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T22:53:16.831755898Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-05-01T22:53:17.476821562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:21.186782912Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-01T22:53:21.288350787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T22:53:21.290045966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:53:22.47657194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:25.541824933Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-01T22:53:27.476505831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:53:27.476529911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:53:27.476540594Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:53:27.476543131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:53:28.837802771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:53:28.842621032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T22:53:29.896774022Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-01T22:53:29.896795599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=20
level=warn ts=2026-05-01T22:53:32.477132575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:53:32.857828712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:53:32.864575078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:53:32.864596334Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T22:53:32.864622718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:53:32.871401209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T22:53:33.231279624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:53:34.255037504Z caller=checkpoint.go:372 msg="writing series" size="5.1 MB"
level=debug ts=2026-05-01T22:53:34.255058691Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=1
level=warn ts=2026-05-01T22:53:37.476485958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:38.606262345Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-01T22:53:42.477141731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:42.961303459Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-01T22:53:43.183468093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:53:43.191112153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:53:47.315732016Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T22:53:47.476457574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:49.732920713Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:53:51.670105149Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T22:53:52.477236318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:53:56.025729679Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=debug ts=2026-05-01T22:53:57.456587119Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:53:57.457625593Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:53:57.477059527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:53:57.477081488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T22:53:57.477085997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:53:57.4770917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:53:57.627118532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T22:53:58.040620689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:53:58.043890931Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T22:54:00.380568408Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=warn ts=2026-05-01T22:54:02.47711888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:54:03.232081428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:54:04.736162943Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-01T22:54:04.736184751Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=7
level=warn ts=2026-05-01T22:54:07.47674025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:09.090171105Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-01T22:54:12.476398616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:13.444572006Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-01T22:54:16.279243678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:54:17.476933605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:17.799657141Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T22:54:22.154859557Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-01T22:54:22.248151682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:54:22.256062253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:54:22.477228384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:26.509406168Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-01T22:54:27.476737347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:54:27.476766223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:54:27.476776704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:54:27.476824793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:30.864160672Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T22:54:32.477038563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:54:33.233145847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T22:54:35.2192396Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-01T22:54:37.476148552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:39.573383113Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T22:54:40.63375975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T22:54:42.476726456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:43.928546377Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-01T22:54:47.476504145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:48.283225916Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-01T22:54:48.827748456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:54:49.414414405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:54:49.418732555Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T22:54:50.786476755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38968"
level=warn ts=2026-05-01T22:54:52.476770924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:54:52.638549715Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-01T22:54:54.41599086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44838"
level=debug ts=2026-05-01T22:54:56.993175293Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=debug ts=2026-05-01T22:54:56.993197409Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-01T22:54:57.457030563Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:54:57.458080417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T22:54:57.476499352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:54:57.476510836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:54:57.47652849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:54:57.476536399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:55:01.348181422Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-01T22:55:02.4762101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:55:03.234635854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:55:05.702737987Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-01T22:55:07.476823678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:10.057560817Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-01T22:55:10.532664591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:55:10.547155983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:55:12.476811132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:14.412929305Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=warn ts=2026-05-01T22:55:17.476847392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:18.767058994Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-01T22:55:22.476285709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:23.122691803Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-01T22:55:23.740549637Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T22:55:24.417579144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45376"
level=info ts=2026-05-01T22:55:27.477131978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:55:27.477134083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:27.477134326Z caller=checkpoint.go:384 msg="flushing series" totalSize="493 kB" series=6
level=debug ts=2026-05-01T22:55:27.477156533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:55:27.477164436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T22:55:27.483053962Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-01T22:55:27.486525234Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010087646s
level=warn ts=2026-05-01T22:55:32.476494804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:55:33.235868964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T22:55:36.326984906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:55:36.33220444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T22:55:37.476382726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:55:40.569928885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T22:55:42.476478068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:55:47.476552713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T22:55:52.476700886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:55:54.419997136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55922"
level=info ts=2026-05-01T22:55:54.431624176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:55:54.440638848Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:55:56.099424241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:55:57.457397866Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:55:57.458443759Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:55:57.476832644Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-01T22:55:57.476855774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:55:57.476859581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:55:57.476881067Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:55:57.476897251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T22:55:57.477038813Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-01T22:55:57.4770518Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T22:55:57.477077024Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.378µs success=true
level=debug ts=2026-05-01T22:55:57.477289999Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-01T22:56:01.832700514Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-01T22:56:02.476400151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:56:03.237032085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T22:56:04.99835935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33068"
level=debug ts=2026-05-01T22:56:06.187166368Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-01T22:56:07.476420735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:10.542595554Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=warn ts=2026-05-01T22:56:12.47710131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:14.897469357Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-01T22:56:17.476277243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:17.698712813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T22:56:19.252248182Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-01T22:56:20.790847466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48308"
level=warn ts=2026-05-01T22:56:22.476982489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:22.819469925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:56:23.606605256Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-01T22:56:26.746514915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:56:26.750186689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T22:56:27.476242273Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:56:27.476249605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:27.476265528Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:56:27.476273491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:56:27.961615294Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T22:56:32.316616723Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T22:56:32.476679535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:56:33.237983009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:56:36.671427449Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-01T22:56:37.476318918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:41.026206107Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T22:56:42.476471211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:56:43.420831358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:56:43.42859906Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:56:45.381351562Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-01T22:56:47.476404337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:49.735866144Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T22:56:52.476742289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:56:54.090511004Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-01T22:56:57.277713678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:56:57.457705279Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T22:56:57.45810097Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:56:57.476500563Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:56:57.476514254Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T22:56:57.476506903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:56:57.476521446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:56:58.445170509Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T22:57:02.476161031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:02.800397356Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=debug ts=2026-05-01T22:57:02.800417671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=16
ts=2026-05-01T22:57:03.238845104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T22:57:07.15552767Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-01T22:57:07.476684753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:11.509580474Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T22:57:12.47634695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:15.864573875Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T22:57:17.47664154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:19.239145562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T22:57:19.240817684Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T22:57:20.219795838Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-01T22:57:22.476974328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:57:23.601460382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:57:23.604701316Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T22:57:24.426070077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33310"
level=debug ts=2026-05-01T22:57:24.575031124Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-01T22:57:25.195569202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T22:57:27.47671594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:57:27.476728615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:27.476743858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:57:27.476753337Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:57:28.929212944Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-01T22:57:29.957461342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:57:29.965215068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T22:57:32.477246572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:57:33.240594127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:57:33.284739532Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-01T22:57:37.477067924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:37.639649009Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-01T22:57:41.994005665Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T22:57:42.476984487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:46.34869309Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T22:57:47.476652647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:50.704019972Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T22:57:52.476822629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:57:54.428289251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60440"
level=debug ts=2026-05-01T22:57:55.058621674Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T22:57:57.457578341Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T22:57:57.457810979Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:57:57.476182855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:57:57.476202383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:57:57.476210982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:57:57.476226105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:57:58.501843982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:57:59.413084135Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-01T22:58:02.476337181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:58:03.242439126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:58:03.767839512Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-01T22:58:07.476952712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:08.123061905Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-01T22:58:11.989410135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T22:58:12.477202035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:12.478269994Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-01T22:58:15.599350598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:58:15.603617067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T22:58:16.833109639Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-01T22:58:17.476757831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:58:20.798170823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40662"
level=debug ts=2026-05-01T22:58:21.187559858Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-01T22:58:22.47678093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:58:23.502091155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:58:23.508577062Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T22:58:24.429886917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46282"
level=debug ts=2026-05-01T22:58:25.542731636Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-01T22:58:27.476665146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:58:27.476672568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:27.476687728Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:58:27.476695973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T22:58:29.897116211Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-01T22:58:29.897139336Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=20
level=warn ts=2026-05-01T22:58:32.477406499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:58:33.243831591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:58:33.746158179Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T22:58:34.255576024Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-01T22:58:34.255599068Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=1
ts=2026-05-01T22:58:35.00516089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44208"
level=warn ts=2026-05-01T22:58:37.476547809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:38.606559474Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-01T22:58:42.476746583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:42.9621656Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-01T22:58:47.316497845Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T22:58:47.476187298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:58:51.671724361Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T22:58:52.476282624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:58:54.431545463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37688"
level=debug ts=2026-05-01T22:58:56.026846868Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-01T22:58:57.457558016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T22:58:57.457574555Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:58:57.476963975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:58:57.476983445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:58:57.476991683Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:58:57.477026414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T22:58:58.753232398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:58:58.761322659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T22:59:00.381290428Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=debug ts=2026-05-01T22:59:01.067714218Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:59:02.476638148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:59:03.2453471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:59:04.736825024Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-01T22:59:04.736848554Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=7
ts=2026-05-01T22:59:05.006824303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58730"
level=debug ts=2026-05-01T22:59:05.778965842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T22:59:07.476261105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:09.091173011Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-01T22:59:12.477110883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:13.446004818Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-01T22:59:17.476258187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:17.800158783Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T22:59:18.50165161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T22:59:18.509847358Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T22:59:22.155764068Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-01T22:59:22.476351434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:59:24.433160473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55208"
level=debug ts=2026-05-01T22:59:26.510144626Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-01T22:59:27.476409354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T22:59:27.476444923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:59:27.476453143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T22:59:27.476496129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:30.864459032Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-01T22:59:30.938668273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T22:59:32.476989446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T22:59:33.246498656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:59:35.220088937Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=warn ts=2026-05-01T22:59:37.476840749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:39.574338328Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-01T22:59:42.476979478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:43.929199666Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-01T22:59:47.476966373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:48.284654876Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-01T22:59:48.579578806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:59:48.583777978Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:59:48.583802456Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T22:59:48.583821901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T22:59:48.586479208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T22:59:52.477183597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:52.639306062Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-01T22:59:54.435162674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41162"
level=debug ts=2026-05-01T22:59:56.993890587Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=debug ts=2026-05-01T22:59:56.993913195Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=debug ts=2026-05-01T22:59:57.457197192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T22:59:57.458236491Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T22:59:57.47659819Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T22:59:57.47660812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T22:59:57.476617003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T22:59:57.476624996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:00:00.16612839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:00:00.851071359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:00:00.855110726Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:00:01.348624257Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-01T23:00:02.380207724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:00:02.387004832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:00:02.387027314Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:00:02.387051508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:00:02.393195661Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:00:02.476400843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:00:03.24785517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:00:05.01024265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35982"
level=debug ts=2026-05-01T23:00:05.548187318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:00:05.703593793Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-01T23:00:07.476742461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:10.05851085Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:00:12.47711875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:14.41397107Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=info ts=2026-05-01T23:00:16.757747445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:00:16.769710333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:00:17.476543059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:18.768388282Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=warn ts=2026-05-01T23:00:22.476764035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:23.123234174Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-01T23:00:26.003211829Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:00:26.422327534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:00:26.427199041Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:00:27.47697309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:00:27.47701649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:00:27.477037807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:00:27.47704953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:00:27.47811839Z caller=checkpoint.go:384 msg="flushing series" totalSize="642 kB" series=6
level=info ts=2026-05-01T23:00:27.484385108Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-01T23:00:27.485930155Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008692957s
level=warn ts=2026-05-01T23:00:32.47619059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:00:33.249468295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:00:36.824929627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:00:36.832545065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:00:37.476976173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:40.789847395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:00:42.476734049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:00:47.476409499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:00:50.805574541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42048"
level=warn ts=2026-05-01T23:00:52.476306493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:00:54.710436011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:00:57.457618016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:00:57.457863076Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-01T23:00:57.460155569Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-01T23:00:57.460198387Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975195 n=1
level=debug ts=2026-05-01T23:00:57.460238164Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975195 n=1
level=info ts=2026-05-01T23:00:57.476270965Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:00:57.476279628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:00:57.476297796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:00:57.476306416Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:00:57.476306349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:00:57.476390615Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-01T23:00:57.476398988Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:00:57.476415577Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.378µs success=true
level=debug ts=2026-05-01T23:00:57.476587385Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=debug ts=2026-05-01T23:01:01.832197543Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-01T23:01:02.476825351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:01:03.251013017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:01:06.186630248Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-01T23:01:07.477050522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:10.542207581Z caller=checkpoint.go:372 msg="writing series" size="448 kB"
level=warn ts=2026-05-01T23:01:12.476979191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:14.897013757Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-01T23:01:17.476432137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:01:18.33825995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:01:18.347268685Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:01:18.981171193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:01:19.251228143Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-01T23:01:22.476614433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:22.708904219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:01:22.710850416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:01:23.606207795Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=debug ts=2026-05-01T23:01:23.606229254Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-01T23:01:24.001163134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:01:24.004442256Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:01:27.476515407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:01:27.476541593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:01:27.47653442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:01:27.476551778Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:01:27.960564648Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-01T23:01:32.316077179Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T23:01:32.476840271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:01:33.252195894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:01:35.01451938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40602"
level=debug ts=2026-05-01T23:01:36.670669746Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-01T23:01:37.476286762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:41.02518195Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-01T23:01:42.476785618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:45.380173163Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-01T23:01:47.47695905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:49.734914037Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:01:52.476460673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:01:53.187055469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:01:54.089714836Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-01T23:01:57.457736219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:01:57.458231067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:01:57.476662337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:01:57.476679091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:01:57.476671873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:01:57.476687435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:01:58.444561974Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:02:00.767750379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:02:02.476734378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:02.800329463Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
ts=2026-05-01T23:02:03.253745842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:02:06.393059138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:02:06.400658137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:02:07.154336276Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-01T23:02:07.47646181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:11.508909442Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-01T23:02:12.476853399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:15.8641759Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-01T23:02:17.476813533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:02:19.561614317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:02:19.567497781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:02:20.219063243Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-01T23:02:22.476647619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:22.819036936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:02:24.445365097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53432"
level=debug ts=2026-05-01T23:02:24.574497447Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-01T23:02:27.476764786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:02:27.476792234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:27.476805256Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:02:27.476815415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:02:28.928772814Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:02:32.476501501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:02:33.255135137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:02:33.283806293Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-01T23:02:37.476554619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:37.638305479Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-01T23:02:37.638326505Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=17
level=debug ts=2026-05-01T23:02:41.993360567Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:02:42.476442615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:46.348580773Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T23:02:47.476862983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:50.702480181Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-01T23:02:50.928066843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:02:50.942930192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:02:52.476877444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:55.058048706Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-01T23:02:57.457310021Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:02:57.458354681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:02:57.47681717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:02:57.476853388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:02:57.476861637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:02:57.476966724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:02:57.843306472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:02:59.392403448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:02:59.412455857Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-01T23:03:02.476866215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:03:03.257314132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:03:03.767620263Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-01T23:03:07.476532075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:08.12264501Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=debug ts=2026-05-01T23:03:12.477138535Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-01T23:03:12.477139185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:03:15.435983187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="234 kB" avg_comp="234 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:03:15.440851608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:03:16.832222285Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:03:17.476444392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:21.186558874Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-01T23:03:22.477218943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:25.542110672Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-01T23:03:27.476268783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:03:27.476297299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:03:27.476308136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:03:27.476307967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:27.712018619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:03:29.896881873Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-01T23:03:32.476817131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:03:33.259679045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:03:34.255295824Z caller=checkpoint.go:372 msg="writing series" size="7.0 MB"
level=debug ts=2026-05-01T23:03:34.255321685Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.7 MB" series=13
level=warn ts=2026-05-01T23:03:37.476615207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:38.606214613Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:03:42.476619726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:42.961518953Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-05-01T23:03:43.345569411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:03:43.356745494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:03:47.316193344Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-01T23:03:47.476959895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:51.670465929Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-01T23:03:52.476757253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:56.025159988Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-05-01T23:03:57.457594579Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:03:57.457889786Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:03:57.476277423Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:03:57.47629908Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:03:57.476306382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:03:57.476338865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:03:58.529239524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:03:59.18075431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.5 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:03:59.184322811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:04:00.380353695Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=debug ts=2026-05-01T23:04:00.380377595Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-01T23:04:01.071194257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:04:02.476479244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:04:03.260989232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:04:04.736302514Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-01T23:04:04.736335559Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=warn ts=2026-05-01T23:04:07.476987959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:09.09001495Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=warn ts=2026-05-01T23:04:12.476606353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:13.44516897Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-01T23:04:17.476466697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:17.799259393Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:04:22.154693701Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-01T23:04:22.476709999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:26.510111884Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=info ts=2026-05-01T23:04:27.476522926Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:04:27.476559516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:04:27.476568198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:04:27.476563569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:28.925789839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:04:30.86417617Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:04:32.477175818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:04:33.263113011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:04:35.024539837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44116"
level=debug ts=2026-05-01T23:04:35.219420824Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=warn ts=2026-05-01T23:04:37.476970767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:39.359312532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:04:39.574335777Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:04:39.574364474Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-01T23:04:39.632400962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:04:39.640999812Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:04:42.477141317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:43.928643524Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T23:04:47.476191584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:48.283532576Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-01T23:04:52.476821933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:04:52.638227836Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-01T23:04:54.264483467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:04:54.269743151Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:04:54.456271382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39772"
level=debug ts=2026-05-01T23:04:57.000011877Z caller=checkpoint.go:372 msg="writing series" size="9.3 MB"
level=debug ts=2026-05-01T23:04:57.000036445Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=4
level=debug ts=2026-05-01T23:04:57.457518843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:04:57.457558489Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:04:57.477138021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:04:57.477171421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:04:57.477179724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:04:57.477170571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:01.348156703Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-01T23:05:02.47714281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:05:03.265627894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:05:03.518367607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:05:05.026246432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33052"
level=debug ts=2026-05-01T23:05:05.703157614Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-01T23:05:07.476253209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:10.057908909Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:05:12.476782637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:14.413015937Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-05-01T23:05:17.476292695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:18.767201877Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
ts=2026-05-01T23:05:20.820533026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40990"
level=warn ts=2026-05-01T23:05:22.476642781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:23.122579272Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-01T23:05:26.256317847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:05:26.265796333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:05:27.476369742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:05:27.476388151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:27.476415462Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:05:27.476427523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:05:27.477479398Z caller=checkpoint.go:384 msg="flushing series" totalSize="548 kB" series=6
level=info ts=2026-05-01T23:05:27.784399157Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-01T23:05:27.785530937Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.308985587s
level=warn ts=2026-05-01T23:05:32.47681755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:33.053648512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:05:33.266716485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=warn ts=2026-05-01T23:05:37.476708872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:38.846782967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:05:38.84842687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T23:05:42.476725254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:05:47.476417647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:05:51.145309313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:05:51.149918019Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:05:52.477053697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:05:57.457767881Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:05:57.457787385Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:05:57.477015545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-01T23:05:57.477032261Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-01T23:05:57.477053508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:05:57.477065828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:05:57.477145761Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-01T23:05:57.477153961Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:05:57.47716983Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.095µs success=true
level=warn ts=2026-05-01T23:05:57.477184394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:05:57.477357442Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-01T23:06:00.896021736Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-01T23:06:02.476598487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:06:03.268782403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:06:03.552760193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:06:04.313858267Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-01T23:06:05.029745032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33720"
level=warn ts=2026-05-01T23:06:07.476421233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:07.730841597Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-01T23:06:11.148968697Z caller=checkpoint.go:372 msg="writing series" size="644 kB"
level=warn ts=2026-05-01T23:06:12.476661486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:06:12.850499968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:06:12.860224075Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:06:12.860249365Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:06:12.860273621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:06:12.869964661Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:06:14.566531968Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-01T23:06:17.476524863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:17.984342668Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:06:21.401679435Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-01T23:06:22.476723949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:06:24.489462218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:06:24.499966078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:06:24.819848672Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-01T23:06:24.819876117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-01T23:06:27.476410595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:06:27.476438201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:06:27.47644644Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:06:27.476461774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:27.968064323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:06:28.237185511Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=debug ts=2026-05-01T23:06:29.33565249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:06:31.655147077Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-01T23:06:32.476959062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:06:33.270458289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:06:35.072933577Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=info ts=2026-05-01T23:06:35.234177681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:06:35.243390972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:06:35.243417479Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:06:35.243455961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:06:35.246549119Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:06:37.476326316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:38.490305626Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-01T23:06:41.908112126Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-01T23:06:42.476718055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:45.326488922Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-01T23:06:45.522154388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:06:45.536520553Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:06:47.476888417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:48.743691657Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:06:52.161131588Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:06:52.476717873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:06:52.84116418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:06:52.846069687Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:06:53.923555014Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:06:55.579282469Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:06:57.457570494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:06:57.457573782Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:06:57.476107769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:06:57.476127039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:06:57.476134927Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:06:57.476129813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:06:58.997150708Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:07:02.41436405Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-01T23:07:02.414387656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-01T23:07:02.476484632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:07:03.272775707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:07:05.033684173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43550"
level=debug ts=2026-05-01T23:07:05.832737375Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-01T23:07:07.477092807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:09.250009098Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-01T23:07:12.476965156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:12.668088174Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-01T23:07:14.692198647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:07:14.698772686Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:07:16.085845417Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-01T23:07:17.476935765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:19.50322618Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=warn ts=2026-05-01T23:07:22.476491107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:22.846870773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:07:22.921448795Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-01T23:07:25.872889572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:07:25.881264256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:07:26.338556139Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=debug ts=2026-05-01T23:07:27.465352384Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:07:27.476605035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:07:27.476642279Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:07:27.476650946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:07:27.476646617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:29.756462523Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:07:32.477108573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:33.174567852Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
ts=2026-05-01T23:07:33.274188853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:07:36.593312521Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-01T23:07:37.476993432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:40.009250603Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:07:42.476725739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:43.42719678Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-01T23:07:46.84525101Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-01T23:07:47.476143071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:50.26251082Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-01T23:07:50.262538068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=14
level=warn ts=2026-05-01T23:07:52.476844441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:53.680202831Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T23:07:56.26618891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:07:56.271987712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:07:57.098335483Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-01T23:07:57.456877439Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:07:57.457958514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:07:57.476591182Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:07:57.476623196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:07:57.476635085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:07:57.476663016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:07:59.537161986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:08:00.516149258Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T23:08:02.476443275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:08:03.275766699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:08:03.933808508Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-01T23:08:07.351753715Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-01T23:08:07.477129465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:08:07.916822593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:08:07.927409696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:08:10.769393211Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-01T23:08:12.476482842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:14.187044312Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-01T23:08:15.872118141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:08:17.476895779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:17.604308641Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-01T23:08:21.022105998Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-01T23:08:22.476836215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:24.440140333Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-01T23:08:26.519569629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:08:27.476661537Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:08:27.476689657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:08:27.47669775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:08:27.476714783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:27.861491257Z caller=checkpoint.go:372 msg="writing series" size="7.5 MB"
level=debug ts=2026-05-01T23:08:27.861512592Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=11
level=debug ts=2026-05-01T23:08:31.275309924Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:08:32.476995073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:08:33.278020572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:08:34.693232153Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=warn ts=2026-05-01T23:08:37.476965291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:38.110888057Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-01T23:08:41.528236553Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-01T23:08:42.476796137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:44.946139683Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
level=debug ts=2026-05-01T23:08:44.946169712Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-01T23:08:47.476244399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:48.364227374Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=debug ts=2026-05-01T23:08:51.782306708Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-01T23:08:51.782340073Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-01T23:08:52.476299303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:08:54.287713421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:08:54.294188384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:08:54.475220957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46636"
level=debug ts=2026-05-01T23:08:55.199331984Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-01T23:08:57.362725532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:08:57.370251938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:08:57.456803877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:08:57.457905588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:08:57.47643612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:08:57.476456981Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:08:57.476467909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:08:57.476467599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:08:58.617544943Z caller=checkpoint.go:372 msg="writing series" size="395 kB"
level=debug ts=2026-05-01T23:08:59.322442899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:09:02.03505805Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-01T23:09:02.476280999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:09:03.279771218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:09:05.452577745Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-01T23:09:07.476888754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:08.870776669Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-05-01T23:09:12.22865405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:09:12.288829407Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-01T23:09:12.477102539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:15.707430709Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T23:09:17.476500188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:19.123011776Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:09:22.477100248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:22.541406938Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-01T23:09:25.959282621Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=debug ts=2026-05-01T23:09:25.95930408Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-01T23:09:27.477127716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:09:27.47713531Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:09:27.477157807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:09:27.477166378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:09:28.779396166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:09:29.376125142Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:09:32.476961212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:32.797427366Z caller=checkpoint.go:372 msg="writing series" size="405 kB"
ts=2026-05-01T23:09:33.281298222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:09:35.738336156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:09:35.743158969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:09:36.212438859Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-01T23:09:37.477049336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:39.629634157Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:09:42.476519869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:43.047367334Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-01T23:09:46.465709101Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-01T23:09:47.47701593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:49.883387149Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-01T23:09:51.357859806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:09:51.372542227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:09:52.476998666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:09:53.300420893Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-01T23:09:54.63649394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:09:56.729555286Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-01T23:09:56.729577784Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=9
level=info ts=2026-05-01T23:09:57.457655638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:09:57.457654787Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:09:57.476179792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:09:57.476196746Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:09:57.476206268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:09:57.476227709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:00.136503098Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-01T23:10:02.477166412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:03.039870197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:10:03.041509521Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:10:03.283072416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:10:03.554077918Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
ts=2026-05-01T23:10:05.043668445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38958"
level=debug ts=2026-05-01T23:10:06.971203436Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-01T23:10:07.476802523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:10.389683127Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-01T23:10:12.476658662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:13.807120803Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T23:10:17.224868933Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=info ts=2026-05-01T23:10:17.297173972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:10:17.303190985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:10:17.47693096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:20.642451191Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-05-01T23:10:20.840053122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38526"
level=debug ts=2026-05-01T23:10:21.357263624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:10:22.476761441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:24.059976538Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-01T23:10:27.476385661Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:10:27.476414517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:10:27.476407674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:10:27.476422463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:10:27.477603828Z caller=checkpoint.go:384 msg="flushing series" totalSize="790 kB" series=8
level=info ts=2026-05-01T23:10:27.733281597Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-01T23:10:27.737349547Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.26004202s
level=warn ts=2026-05-01T23:10:32.477156259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:10:33.28489198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:10:34.107069914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:10:34.116563137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T23:10:35.044575768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42350"
level=warn ts=2026-05-01T23:10:37.477129091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:10:42.47708215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:10:47.476918568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:50.136059518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:10:50.841687818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39302"
level=warn ts=2026-05-01T23:10:52.477126313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:10:55.395848037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:10:57.457014049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:10:57.45804466Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:10:57.476596599Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-01T23:10:57.47660709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:10:57.476629916Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:10:57.476646966Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:10:57.476654864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:10:57.476700088Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-01T23:10:57.47670789Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:10:57.47672271Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.441µs success=true
level=debug ts=2026-05-01T23:10:57.476910395Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-01T23:11:00.895247306Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-01T23:11:02.476409923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:11:03.286914671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:11:04.312560412Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-01T23:11:06.358200519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="120 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:11:06.362287403Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:11:07.476307313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:07.731101325Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-01T23:11:11.148355957Z caller=checkpoint.go:372 msg="writing series" size="630 kB"
level=warn ts=2026-05-01T23:11:12.476458678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:14.565859116Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-01T23:11:17.476579949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:17.983714603Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-01T23:11:18.448039875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:11:20.84271713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41844"
level=debug ts=2026-05-01T23:11:21.401286784Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T23:11:22.476431914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:24.819642107Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:11:27.405757019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:11:27.416678995Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:11:27.47619405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:11:27.476218886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:11:27.476213797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:11:27.4762266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:11:28.237490039Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=debug ts=2026-05-01T23:11:28.237512855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-05-01T23:11:31.655228436Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-01T23:11:32.477143614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:11:33.288452676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:11:35.046297711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60466"
level=debug ts=2026-05-01T23:11:35.072161993Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T23:11:37.476567673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:38.489905874Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-01T23:11:41.908263509Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:11:41.985359408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:11:42.476180351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:45.325672353Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=warn ts=2026-05-01T23:11:47.476985681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:47.531208072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:11:48.74288774Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-01T23:11:50.843823857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48284"
level=info ts=2026-05-01T23:11:50.984195826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.5 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:11:50.988106111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:11:52.160591518Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:11:52.477199041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:11:54.487162056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55646"
level=debug ts=2026-05-01T23:11:55.578569661Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-01T23:11:57.456985276Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:11:57.45806449Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:11:57.476596728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:11:57.476618532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:11:57.476625629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:11:57.476646905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:11:58.996578767Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:12:02.414202508Z caller=checkpoint.go:372 msg="writing series" size="335 kB"
level=warn ts=2026-05-01T23:12:02.476700336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:12:03.290558408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:12:05.832044404Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-01T23:12:07.476253482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:09.24989822Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-01T23:12:12.47712127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:12.667093194Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-01T23:12:12.667112266Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-05-01T23:12:16.084779732Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-01T23:12:17.476808717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:18.853889047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:12:19.503181683Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-01T23:12:19.920068717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:12:19.930980362Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:12:19.93100816Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:12:19.93103877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:12:19.938433003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:12:22.476422494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:22.920254707Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-01T23:12:26.081557823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:12:26.338199624Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=warn ts=2026-05-01T23:12:27.476338017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:12:27.476359612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:12:27.476386568Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:12:27.476397752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:12:29.755633295Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:12:32.476872594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:33.173625333Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
ts=2026-05-01T23:12:33.292267516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:12:36.591577647Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-01T23:12:37.04736875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:12:37.055843959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:12:37.476967625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:40.009597574Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:12:42.476728041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:43.426740398Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-01T23:12:46.844517375Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-01T23:12:47.476299659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:12:48.251068211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:12:48.256784026Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:12:48.256807674Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:12:48.256836548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:12:48.260062435Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:12:50.192669484Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:12:50.262532145Z caller=checkpoint.go:372 msg="writing series" size="529 kB"
level=warn ts=2026-05-01T23:12:52.476466331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:12:53.679665077Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-01T23:12:53.679693896Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
ts=2026-05-01T23:12:54.491135723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60766"
level=debug ts=2026-05-01T23:12:57.098155128Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-01T23:12:57.457142505Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:12:57.458194318Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:12:57.476707208Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:12:57.476730823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:12:57.476743461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:12:57.476805859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:00.515175336Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-01T23:13:02.47633314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:13:03.294061054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:13:03.933814542Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=info ts=2026-05-01T23:13:07.220622248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:13:07.224966096Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:13:07.350631098Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-01T23:13:07.476504813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:10.768426199Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=warn ts=2026-05-01T23:13:12.47616004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:13:13.951327759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:13:13.958704075Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:13:14.186380501Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:13:16.184999578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:13:17.352574269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:13:17.47651051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:17.603933767Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-01T23:13:21.021852213Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-01T23:13:22.476671547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:24.439790602Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-01T23:13:27.476390337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:13:27.476425406Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:13:27.476424352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:13:27.476435769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:13:27.864981027Z caller=checkpoint.go:372 msg="writing series" size="8.0 MB"
level=debug ts=2026-05-01T23:13:27.865002398Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.8 MB" series=10
level=debug ts=2026-05-01T23:13:31.275026646Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:13:32.476666628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:13:33.29561973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:13:34.693535702Z caller=checkpoint.go:372 msg="writing series" size="725 kB"
level=info ts=2026-05-01T23:13:34.750272313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:13:34.756704326Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:13:37.476596671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:38.11080417Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-01T23:13:41.527937917Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-01T23:13:42.47681572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:42.570171543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:13:44.945597844Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-05-01T23:13:44.945620865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-01T23:13:47.476972784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:48.363401376Z caller=checkpoint.go:372 msg="writing series" size="701 kB"
level=debug ts=2026-05-01T23:13:51.783817191Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:13:51.783846074Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=warn ts=2026-05-01T23:13:52.476896726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:13:54.495125449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53610"
level=debug ts=2026-05-01T23:13:55.201245448Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=info ts=2026-05-01T23:13:57.457641681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:13:57.457646513Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:13:57.477130989Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:13:57.477161249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:13:57.477172532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:13:57.477177645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:13:58.617472152Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-01T23:14:02.034510272Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=debug ts=2026-05-01T23:14:02.034546741Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-01T23:14:02.476766127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:14:03.299210524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:14:05.056976862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52032"
level=debug ts=2026-05-01T23:14:05.452399496Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:14:06.826166016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:14:06.828829128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:14:06.830540428Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:14:06.832610154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:14:07.476433426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:08.870078526Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-01T23:14:12.288738459Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=warn ts=2026-05-01T23:14:12.476322291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:15.705391684Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T23:14:17.476769325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:18.085599941Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:14:19.122784486Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:14:22.477250414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:22.540576308Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-01T23:14:25.958573588Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=debug ts=2026-05-01T23:14:25.958595591Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-01T23:14:27.476191599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:14:27.476216679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:14:27.476226884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:14:27.476224216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:29.376577408Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:14:32.477007836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:32.794043731Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
ts=2026-05-01T23:14:33.30126189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:14:34.432094286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="190 MB" avg_uncomp="21 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:14:34.441906606Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:14:35.058657097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37602"
level=debug ts=2026-05-01T23:14:36.212350204Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-01T23:14:37.476480411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:39.629311766Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:14:42.476846392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:43.047448155Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-01T23:14:46.464474283Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-01T23:14:47.476469569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:48.930314947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:14:49.882259564Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-01T23:14:50.343462793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:14:50.345528177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:14:50.353455327Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-01T23:14:50.853813265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34754"
level=warn ts=2026-05-01T23:14:52.476924785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:14:53.300776216Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-01T23:14:54.500978488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59542"
level=debug ts=2026-05-01T23:14:56.750485406Z caller=checkpoint.go:372 msg="writing series" size="26 MB"
level=debug ts=2026-05-01T23:14:56.750516527Z caller=checkpoint.go:384 msg="flushing series" totalSize="27 MB" series=9
level=info ts=2026-05-01T23:14:57.457706435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:14:57.457715062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=warn ts=2026-05-01T23:14:57.476142772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:14:57.47610332Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:14:57.476237026Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:14:57.476245887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:15:00.135985177Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-01T23:15:02.477079323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:15:03.303221798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:15:03.553173133Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-01T23:15:04.193032614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:15:04.203911614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:15:06.971524518Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-01T23:15:07.476286151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:15:10.028015463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:15:10.038574235Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:15:10.391168446Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-01T23:15:12.476747087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:13.806975911Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T23:15:17.224906212Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-01T23:15:17.476386717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:20.642601252Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-01T23:15:22.47636844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:22.785865307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:15:24.059916409Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-01T23:15:27.476267665Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:15:27.476276141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:27.47629684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:15:27.4763063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:15:27.477379488Z caller=checkpoint.go:384 msg="flushing series" totalSize="947 kB" series=8
level=info ts=2026-05-01T23:15:27.482130732Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-01T23:15:27.483217544Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006355552s
level=warn ts=2026-05-01T23:15:32.476376425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:15:33.17020091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="26 MB" avg_comp="2.6 MB" total_uncomp="214 MB" avg_uncomp="21 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:15:33.175920515Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:15:33.305223011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:15:35.061853261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43004"
level=warn ts=2026-05-01T23:15:37.476761207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:15:38.058437355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:15:38.0664044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:15:42.476473504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:43.837646007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:15:47.476296552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:15:50.857900207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55970"
level=warn ts=2026-05-01T23:15:52.476303652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:15:52.528428784Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:15:56.886877249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:15:56.900726818Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:15:57.457562544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:15:57.457984636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-01T23:15:57.459336872Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-01T23:15:57.459379194Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975196 n=1
level=debug ts=2026-05-01T23:15:57.459431884Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975196 n=1
level=info ts=2026-05-01T23:15:57.476475083Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-01T23:15:57.476485314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:15:57.476510263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:15:57.476522134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:15:57.476532485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:15:57.476581683Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-01T23:15:57.476591099Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:15:57.476618584Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=22.342µs success=true
level=debug ts=2026-05-01T23:15:57.47681937Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=debug ts=2026-05-01T23:16:00.89474042Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-01T23:16:02.476846477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:16:03.306980212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:16:04.31323697Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-01T23:16:07.476577595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:07.730508289Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-01T23:16:11.147975969Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=warn ts=2026-05-01T23:16:12.476473362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:14.565632438Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-01T23:16:14.964489187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="260 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:16:14.969078783Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:16:17.476511724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:17.703629128Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:16:17.983319698Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-01T23:16:21.402021405Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=debug ts=2026-05-01T23:16:21.402042859Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-01T23:16:22.476380968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:24.819673254Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:16:27.240292767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:16:27.249155373Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:16:27.476195053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:16:27.476204481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:27.476221202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:16:27.476228981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:16:28.237366091Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=debug ts=2026-05-01T23:16:31.654607604Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-01T23:16:32.476357228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:16:33.309161993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:16:35.064307579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48496"
level=debug ts=2026-05-01T23:16:35.071915782Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T23:16:37.476962887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:38.491570405Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-01T23:16:38.759482534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:16:38.768777551Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:16:41.907394118Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-01T23:16:42.476457778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:44.526902822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:16:45.326007067Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-01T23:16:45.382298903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:16:47.476537887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:48.74319693Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:16:52.160959989Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:16:52.477135758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:55.579102486Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-05-01T23:16:55.579123323Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-01T23:16:57.457828291Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:16:57.458337643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:16:57.476672889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:16:57.476693516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:16:57.476702015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:16:57.476764718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:16:58.996633163Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:17:02.414304727Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-01T23:17:02.477055725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:17:03.310153591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-01T23:17:04.104515148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="32 MB" avg_comp="2.7 MB" total_uncomp="260 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:17:04.109707746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:17:05.065379516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41556"
level=debug ts=2026-05-01T23:17:05.83204718Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-01T23:17:07.476907642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:09.249476255Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-01T23:17:11.165933713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:17:11.171652368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:17:12.476988306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:12.667080363Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-01T23:17:16.085335538Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-01T23:17:17.476492245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:19.502409504Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-01T23:17:19.833153485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:17:22.476381887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:22.92065808Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-01T23:17:24.516625719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39242"
level=debug ts=2026-05-01T23:17:26.337979129Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:17:26.549001679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:17:26.562351504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:17:27.400269039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:17:27.477104806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:17:27.477128377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:17:27.47713606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:17:27.477139396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:29.756239183Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:17:32.476881749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:33.173627614Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
ts=2026-05-01T23:17:33.311296178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:17:35.066435484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44816"
level=debug ts=2026-05-01T23:17:36.591091567Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-01T23:17:37.476396778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:40.009399487Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:17:42.477188463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:43.426616302Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-01T23:17:46.844787374Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-01T23:17:46.844810417Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-05-01T23:17:47.476757658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:50.263025108Z caller=checkpoint.go:372 msg="writing series" size="783 kB"
level=warn ts=2026-05-01T23:17:52.476866215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:17:53.679992514Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-01T23:17:53.757255504Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:17:53.75730124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:17:53.763335853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:17:57.097544803Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=info ts=2026-05-01T23:17:57.40327546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="35 MB" avg_comp="2.7 MB" total_uncomp="284 MB" avg_uncomp="22 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:17:57.4078412Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:17:57.457048437Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:17:57.458105859Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:17:57.476532515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:17:57.476556875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:17:57.476566147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:17:57.476594618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:17:57.972843317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:17:57.979189444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:18:00.515900415Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-01T23:18:02.476407372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:18:03.31259729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:18:03.933122054Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=debug ts=2026-05-01T23:18:03.933146457Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-01T23:18:04.613390071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:18:04.618930512Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:18:07.350249823Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-01T23:18:07.477044128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:10.76863186Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=debug ts=2026-05-01T23:18:10.899877352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:18:10.901809435Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T23:18:12.476184889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:14.185874525Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-05-01T23:18:17.477067111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:17.604558312Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:18:21.021843467Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-01T23:18:22.476564042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:24.439680294Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-01T23:18:24.583415745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:18:24.596860562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:18:27.476256397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:18:27.47628208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:18:27.476290326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:18:27.476341994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:27.865074396Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-01T23:18:27.86509842Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=7
level=debug ts=2026-05-01T23:18:29.248141011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:18:31.274381827Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:18:32.476711163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:18:33.31368682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:18:34.694168947Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-01T23:18:34.694191111Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-01T23:18:35.654902175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:18:35.662538137Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:18:37.477033651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:38.110406519Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-01T23:18:41.527706411Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-01T23:18:42.47700072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:44.945991358Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-01T23:18:47.476259774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:48.363374529Z caller=checkpoint.go:372 msg="writing series" size="892 kB"
level=debug ts=2026-05-01T23:18:48.363402443Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-01T23:18:50.467003136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="308 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:18:50.47286047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:18:51.743151842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:18:51.749427525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:18:51.783020754Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-01T23:18:51.783040271Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=1
level=warn ts=2026-05-01T23:18:52.476657892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:55.199125605Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-05-01T23:18:57.457262863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:18:57.458322923Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:18:57.476784294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:18:57.476797684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:18:57.47682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:18:57.476838973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:18:58.61733707Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-01T23:18:58.617360854Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=debug ts=2026-05-01T23:19:02.034435243Z caller=checkpoint.go:372 msg="writing series" size="622 kB"
level=debug ts=2026-05-01T23:19:02.351536692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:19:02.476829557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:19:03.315055486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:19:04.76697344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:19:05.070628355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34960"
level=debug ts=2026-05-01T23:19:05.452403435Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-01T23:19:07.477932588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:08.872080921Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-01T23:19:08.872103436Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-01T23:19:09.967025081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:19:09.97874217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:19:09.978775885Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:19:09.978810601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:19:09.990595339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:19:12.289779924Z caller=checkpoint.go:372 msg="writing series" size="854 kB"
level=warn ts=2026-05-01T23:19:12.476864777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:15.705211277Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T23:19:17.476292959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:19.122753671Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:19:22.477041686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:22.540263097Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=debug ts=2026-05-01T23:19:22.540286085Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-01T23:19:23.742442782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:19:23.754596209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:19:25.959032948Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-01T23:19:26.803237141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:19:26.807872568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:19:27.191115418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:19:27.476261751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:19:27.476292104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:19:27.476302358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:19:27.476314362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:29.376023995Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:19:32.476676431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:32.794452797Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=debug ts=2026-05-01T23:19:32.794478693Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-01T23:19:33.316462366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:19:33.607242334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:19:33.614606698Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:19:36.211888431Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-01T23:19:37.47692667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:19:39.060090631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="332 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:19:39.064913396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:19:39.064957297Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:19:39.064998821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="332 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:19:39.068738206Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:19:39.629519355Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:19:42.477454388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:42.975595017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:19:43.046930338Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-01T23:19:46.465114032Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-01T23:19:47.476476774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:19:49.082019687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:19:49.101928344Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:19:49.882516417Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-01T23:19:51.673147476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="332 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:19:51.676302302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:19:52.477035333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:19:53.302478121Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-01T23:19:54.277604559Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:19:56.765514471Z caller=checkpoint.go:372 msg="writing series" size="42 MB"
level=debug ts=2026-05-01T23:19:56.765548042Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=7
level=debug ts=2026-05-01T23:19:57.457120867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:19:57.458164127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:19:57.476525498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:19:57.476556692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:19:57.476566355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:19:57.476602657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:20:00.135961655Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-01T23:20:02.476760017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:20:03.318328087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:20:03.553349346Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-05-01T23:20:05.073671403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48060"
level=debug ts=2026-05-01T23:20:06.971416389Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-01T23:20:07.476134224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:20:10.393810885Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=warn ts=2026-05-01T23:20:12.477207263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:20:13.103715514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="332 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:20:13.110525656Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:20:13.806375786Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T23:20:17.224019931Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=debug ts=2026-05-01T23:20:17.22404483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-01T23:20:17.477022585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:20:18.605209965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:20:18.61207976Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:20:20.641683575Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
ts=2026-05-01T23:20:20.873466394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56252"
level=warn ts=2026-05-01T23:20:22.476438389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:20:22.821959808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:20:22.82587658Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:20:24.059703746Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-01T23:20:24.535510168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48420"
level=info ts=2026-05-01T23:20:27.476103294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:20:27.476140331Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:20:27.476149141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:20:27.476150186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:20:27.47719617Z caller=checkpoint.go:384 msg="flushing series" totalSize="281 kB" series=2
level=info ts=2026-05-01T23:20:27.484029471Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-01T23:20:27.486156528Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009404643s
level=debug ts=2026-05-01T23:20:27.565722526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:20:32.476644952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:20:33.32061527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:20:34.955921123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:20:35.075416336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33036"
level=warn ts=2026-05-01T23:20:37.476851953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:20:40.780179177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:20:40.793336516Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:20:42.477171956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:20:47.47640244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:20:52.476494667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:20:54.537574882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60886"
level=info ts=2026-05-01T23:20:57.457664657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:20:57.457674469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:20:57.476035432Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:20:57.476168145Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-01T23:20:57.476176945Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:20:57.476194462Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.232µs success=true
level=info ts=2026-05-01T23:20:57.476240978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:20:57.476258107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:20:57.476267882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:20:57.476294505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:20:57.47639932Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=debug ts=2026-05-01T23:21:00.810516473Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-01T23:21:01.393819625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:21:01.590568556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.7 MB" total_uncomp="332 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:21:01.59661294Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:21:02.476829851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:21:03.321968342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:21:04.144397441Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=debug ts=2026-05-01T23:21:07.476545399Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-01T23:21:07.476557488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:21:08.429723508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:21:08.437294193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:21:08.437321318Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:21:08.437351224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:21:08.444237697Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:21:10.811765168Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-01T23:21:10.811793986Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-05-01T23:21:12.476989729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:14.143774044Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-01T23:21:17.47621991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:17.477471594Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-01T23:21:20.81104991Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-01T23:21:20.876617884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41716"
level=info ts=2026-05-01T23:21:21.213891646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:21:21.220590577Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:21:22.476715354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:21:23.65514062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:21:23.663980918Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:21:24.143344002Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:21:24.430914701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:21:26.869640841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:21:26.878136962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:21:27.476384718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:21:27.476431582Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:21:27.476442469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:21:27.47649934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:27.476733659Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-01T23:21:27.476751402Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-01T23:21:28.891373866Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:21:30.810928404Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-01T23:21:32.47686733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:21:33.323417861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:21:34.143695241Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=debug ts=2026-05-01T23:21:37.477182932Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-01T23:21:37.477188504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:40.810887934Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-01T23:21:42.476718346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:44.143358772Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-01T23:21:47.476295186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:47.477410412Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-01T23:21:47.887793851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:21:47.902187767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:21:50.809786575Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-01T23:21:50.878257503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38286"
level=warn ts=2026-05-01T23:21:52.477110471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:54.144087099Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-01T23:21:56.982075914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:21:56.985812468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:21:57.45755478Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:21:57.457564228Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:21:57.476996854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:21:57.477018504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:21:57.47702861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:21:57.477021459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:21:57.47707311Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:22:00.810193007Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:22:02.191050292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:22:02.193919832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T23:22:02.476633691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:03.10783696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:22:03.324744494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:22:04.14434726Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=debug ts=2026-05-01T23:22:04.144373176Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-01T23:22:07.476611314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:07.476684624Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-01T23:22:10.810419967Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-01T23:22:12.47648669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:22:13.70235223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:22:13.708702162Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:22:14.144099019Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-05-01T23:22:17.4772094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:17.477327201Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-01T23:22:20.810642991Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-01T23:22:22.47684441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:22:22.752980071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:22:22.763095883Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:22:24.143580156Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
ts=2026-05-01T23:22:24.544156073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44492"
level=info ts=2026-05-01T23:22:27.476928684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:22:27.476973504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:22:27.476983499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:22:27.477019698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:27.477045975Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=debug ts=2026-05-01T23:22:30.455728067Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:22:30.455785514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:22:30.464786015Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:22:30.811049135Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:22:32.477185651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:22:33.327024465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:22:34.143373548Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-01T23:22:35.08237879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43076"
level=warn ts=2026-05-01T23:22:37.476358519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:37.476441907Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-01T23:22:40.810666658Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:22:42.476688454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:42.499372204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:22:42.733307006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:22:42.74072934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:22:42.911514823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:22:42.915507755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:22:43.547897579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:22:43.555320701Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:22:44.143318426Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T23:22:47.476861622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:47.476996818Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-05-01T23:22:47.477013058Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-05-01T23:22:50.810889911Z caller=checkpoint.go:372 msg="writing series" size="854 kB"
ts=2026-05-01T23:22:50.880701612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48142"
level=warn ts=2026-05-01T23:22:52.476918172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:22:54.143744027Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
ts=2026-05-01T23:22:54.5463083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45920"
level=info ts=2026-05-01T23:22:57.457584535Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:22:57.458136754Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-01T23:22:57.476631334Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=info ts=2026-05-01T23:22:57.476652179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:22:57.476671943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:22:57.476654298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:22:57.476682347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:22:58.265067288Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:23:00.80983516Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-01T23:23:00.809859186Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-01T23:23:02.476841227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:23:03.328672448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:23:04.144341684Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=warn ts=2026-05-01T23:23:07.476455551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:07.476494148Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-01T23:23:10.810814481Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-01T23:23:11.194473351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:23:11.202875624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:23:12.476296076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:14.143565074Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=info ts=2026-05-01T23:23:17.428611568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:23:17.437573302Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:23:17.47680615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:17.477103875Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:23:17.477117605Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-01T23:23:18.834872528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:23:18.843649867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:23:20.809972168Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-01T23:23:22.476840901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:24.143315072Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-01T23:23:27.476667161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:23:27.476677722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:27.476691963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:23:27.476699873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:23:27.486871254Z caller=checkpoint.go:372 msg="writing series" size="9.5 MB"
level=debug ts=2026-05-01T23:23:27.486900577Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.6 MB" series=3
level=debug ts=2026-05-01T23:23:30.809802968Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:23:32.476361353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:23:33.224579592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:23:33.232826787Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-01T23:23:33.330799307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:23:33.416254856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:23:34.145263021Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-01T23:23:34.14528618Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=2
ts=2026-05-01T23:23:35.086016778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40418"
level=warn ts=2026-05-01T23:23:37.477127794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:37.47718079Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-01T23:23:39.225343128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:23:40.803113386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:23:40.806785299Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:23:40.809915243Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-01T23:23:42.476515297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:44.144194878Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=warn ts=2026-05-01T23:23:47.476704662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:47.477410021Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-01T23:23:47.477423602Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=debug ts=2026-05-01T23:23:50.81320543Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-01T23:23:50.813229391Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=1
level=warn ts=2026-05-01T23:23:52.476721892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:54.144181888Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=debug ts=2026-05-01T23:23:57.457475338Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:23:57.458295544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:23:57.476983651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:23:57.476995353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:23:57.477014649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:23:57.47702458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:23:57.478082458Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-01T23:23:57.478100784Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-01T23:24:00.567519429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:24:00.573004692Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:24:00.810929733Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-01T23:24:02.476346914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:24:03.050979797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:24:03.058252653Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-01T23:24:03.332146534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:24:04.143200107Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
ts=2026-05-01T23:24:05.087631973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51566"
level=debug ts=2026-05-01T23:24:06.213507665Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:24:07.476540115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:07.476534118Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-01T23:24:10.813249148Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-01T23:24:10.81327525Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=4
level=warn ts=2026-05-01T23:24:12.476748871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:14.144514718Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-01T23:24:16.830057436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:24:16.840519832Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:24:17.476682229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:17.476702845Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:24:18.840726974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:24:18.848778055Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:24:20.810103046Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:24:22.477132224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:24.143997244Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-01T23:24:24.144020639Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-01T23:24:24.300641773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:24:24.303901983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:24:26.879829476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:24:27.476876796Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:24:27.476880468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:27.476903186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:24:27.477114379Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:24:27.477320442Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=debug ts=2026-05-01T23:24:30.810526732Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:24:32.476497138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:24:33.333582683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:24:34.143818465Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=debug ts=2026-05-01T23:24:34.143845436Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-05-01T23:24:35.089396275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35632"
level=warn ts=2026-05-01T23:24:37.47662963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:37.47668338Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:24:40.342269796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:24:40.810634663Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:24:42.476169363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:44.143918721Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-01T23:24:46.765740817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:24:46.773534696Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:24:47.476528645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:47.476993642Z caller=checkpoint.go:372 msg="writing series" size="866 kB"
level=debug ts=2026-05-01T23:24:47.47701038Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-01T23:24:50.81051735Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:24:52.476693346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:54.143091705Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-01T23:24:55.977305104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:24:55.982135961Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:24:55.982157351Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:24:55.982180271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:24:55.985367974Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:24:57.184784718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:24:57.191643586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:24:57.457119185Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:24:57.458170055Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:24:57.47664804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:24:57.476671725Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:24:57.476680385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:24:57.476699024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:24:57.519986845Z caller=checkpoint.go:372 msg="writing series" size="46 MB"
level=debug ts=2026-05-01T23:24:57.520003585Z caller=checkpoint.go:384 msg="flushing series" totalSize="46 MB" series=3
level=debug ts=2026-05-01T23:25:00.810070752Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-01T23:25:02.476458478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:25:03.335142744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:25:04.14391676Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-01T23:25:07.476470259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:25:07.476483174Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-01T23:25:07.704380892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:25:07.711662609Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:25:10.278065113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:25:10.28730625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:25:10.810156469Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=warn ts=2026-05-01T23:25:12.477043571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:25:14.143768287Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T23:25:14.250277672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:25:17.042013898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.7 MB" total_uncomp="355 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:25:17.045461457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:25:17.239499564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:25:17.476397169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:25:17.476582098Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=debug ts=2026-05-01T23:25:17.476596992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-01T23:25:20.810741383Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-05-01T23:25:22.476858527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:25:24.143696053Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
ts=2026-05-01T23:25:24.557421987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46102"
level=debug ts=2026-05-01T23:25:27.476720744Z caller=checkpoint.go:384 msg="flushing series" totalSize="290 kB" series=2
level=warn ts=2026-05-01T23:25:27.476735403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:25:27.476761619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:25:27.476780126Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:25:27.47678875Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:25:27.482407074Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-01T23:25:27.484545805Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008211754s
level=warn ts=2026-05-01T23:25:32.476370247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:25:33.33676573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:25:35.091974513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48612"
level=warn ts=2026-05-01T23:25:37.476455729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:25:39.869429458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:25:40.257501789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:25:40.265450434Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:25:42.476897928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:25:45.997638617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:25:46.006775185Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:25:46.069146962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:25:46.072912655Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:25:47.476844254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:25:50.891853581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40354"
level=warn ts=2026-05-01T23:25:52.477046351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:25:56.362446276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:25:56.370685954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:25:56.370710824Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:25:56.370741514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:25:56.377351548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:25:57.457416835Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:25:57.458468071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:25:57.476996362Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:25:57.477009062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:25:57.47703204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:25:57.477040284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:25:57.477087987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:25:57.477132428Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-01T23:25:57.47714241Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:25:57.477161581Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.397µs success=true
level=debug ts=2026-05-01T23:25:57.477376224Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-01T23:26:00.811196372Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-01T23:26:02.477194601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:26:03.337904382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:26:04.145556037Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=warn ts=2026-05-01T23:26:07.476142611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:07.478314818Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-01T23:26:08.806374899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:26:08.817822565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:26:09.361765052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:26:10.811675549Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-01T23:26:10.811703642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:26:12.47679968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:14.144505395Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-01T23:26:14.192544431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:26:14.194423344Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T23:26:16.270687964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:26:16.274458063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:26:16.274477696Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:26:16.274511399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:26:16.27758906Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:26:17.47702962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:17.478242312Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-01T23:26:20.811511987Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-01T23:26:20.893338437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44802"
level=warn ts=2026-05-01T23:26:22.476860196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:24.144758532Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:26:27.079146275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:26:27.085069487Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:26:27.437624565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:26:27.441723523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:26:27.47649194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:26:27.476520992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:26:27.476529132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:26:27.4765229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:27.478121848Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=debug ts=2026-05-01T23:26:27.478134139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-01T23:26:30.811258159Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-01T23:26:30.983790441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:26:30.991997109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:26:32.260816389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:26:32.267620123Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:26:32.476957351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:26:33.339998061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:26:33.450388127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:26:33.457252394Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:26:34.144170819Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T23:26:37.477070593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:37.47817204Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-01T23:26:40.810912094Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-01T23:26:41.448900342Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:26:42.476729372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:44.145184002Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-01T23:26:47.476282574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:47.478405055Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:26:50.8107928Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-01T23:26:50.895105727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51462"
level=warn ts=2026-05-01T23:26:52.476904376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:26:54.144538681Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-01T23:26:56.418427165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:26:56.424575071Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:26:57.457537762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:26:57.457566273Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T23:26:57.477131309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:26:57.47714913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:26:57.477171525Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:26:57.477180876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:26:57.478288012Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:27:00.810754975Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T23:27:02.476973115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:27:03.341412124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:27:04.144832723Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-01T23:27:04.144866841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-01T23:27:07.4766813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:07.47792024Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-01T23:27:07.796056271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:27:10.811767993Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-01T23:27:12.477134116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:14.144742187Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-01T23:27:14.725661259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:27:17.476201787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:17.478440082Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-01T23:27:18.169642418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:27:18.173355781Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:27:20.81092102Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:27:22.476917894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:27:23.309128809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:27:23.316286357Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:27:24.144732026Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-01T23:27:24.144753089Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-01T23:27:25.070848779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:27:25.077814787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:27:26.036388463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:27:26.04369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:27:26.043712154Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:27:26.04373593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:27:26.049418447Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:27:27.476177967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:27:27.47620253Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:27:27.476210275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:27:27.476208518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:27.478389541Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=debug ts=2026-05-01T23:27:30.811392586Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:27:32.476405868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:27:33.342670592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:27:34.145090222Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-01T23:27:37.476592249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:37.477681109Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-01T23:27:39.786683278Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:27:39.786751316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:27:39.798395167Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:27:40.81078162Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:27:42.476481565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:44.144245379Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-01T23:27:45.776075489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:27:45.782298285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:27:47.476751435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:47.477990782Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-01T23:27:50.812005845Z caller=checkpoint.go:372 msg="writing series" size="911 kB"
level=debug ts=2026-05-01T23:27:50.812029622Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
ts=2026-05-01T23:27:50.897869688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50310"
level=warn ts=2026-05-01T23:27:52.476479877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:27:54.144488488Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T23:27:56.182996408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="46 MB" avg_comp="2.7 MB" total_uncomp="379 MB" avg_uncomp="22 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:27:56.187396957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:27:57.457107075Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:27:57.45815656Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:27:57.47663871Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:27:57.47666516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:27:57.47664972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:27:57.476673248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:27:57.477783794Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=debug ts=2026-05-01T23:27:58.145542875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:27:59.616704013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:27:59.62651347Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:28:00.810857934Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-01T23:28:02.47652092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:28:03.34405432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:28:04.144831309Z caller=checkpoint.go:372 msg="writing series" size="361 kB"
level=warn ts=2026-05-01T23:28:07.476847262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:07.478026987Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-01T23:28:09.040604707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:28:09.044757816Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:28:10.373834141Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:28:10.811191944Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-01T23:28:12.476981805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:14.144817436Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=warn ts=2026-05-01T23:28:17.476151239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:17.478460877Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:28:17.47847355Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-01T23:28:20.810934748Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-01T23:28:21.206981851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:28:21.216608987Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:28:21.539698726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:28:21.545591736Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:28:21.809141122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:28:21.816335997Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:28:22.477041411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:28:23.217081288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:28:23.225692381Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:28:24.144681081Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-01T23:28:27.477051457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:28:27.477081245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:28:27.477091784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:28:27.477082105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:27.488776249Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-01T23:28:27.488791822Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=debug ts=2026-05-01T23:28:30.811603928Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:28:32.47699054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:28:33.345643171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:28:34.152769073Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-01T23:28:34.152800425Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.4 MB" series=2
level=debug ts=2026-05-01T23:28:36.921249525Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:28:37.47626137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:37.477362537Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-01T23:28:40.810812175Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-01T23:28:42.477185352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:44.144854374Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=warn ts=2026-05-01T23:28:47.476362234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:47.478123252Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-01T23:28:47.478139763Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-05-01T23:28:50.644270839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:28:50.648038222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:28:50.813069602Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-01T23:28:50.81309206Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=1
level=warn ts=2026-05-01T23:28:52.47699831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:53.004427338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:28:54.14482841Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=debug ts=2026-05-01T23:28:57.457013847Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:28:57.45806471Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:28:57.476614791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:28:57.476632105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:28:57.476640827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:28:57.476648241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:28:57.478423145Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-01T23:28:57.478435213Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=debug ts=2026-05-01T23:29:00.811653598Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-01T23:29:02.476919297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:29:02.981149859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:29:02.985775516Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-01T23:29:03.347197628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:29:04.144509559Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-01T23:29:06.169269713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:29:06.176662573Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:29:07.366190823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:29:07.37379566Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:29:07.476234024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:07.4783809Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-01T23:29:08.049443289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:29:09.55156936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:29:09.557736278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:29:10.816557036Z caller=checkpoint.go:372 msg="writing series" size="4.7 MB"
level=debug ts=2026-05-01T23:29:10.816581175Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.7 MB" series=4
level=warn ts=2026-05-01T23:29:12.477089619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:29:13.331132281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:29:13.339124723Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:29:13.339146243Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:29:13.339169864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:29:13.346581395Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:29:14.145265429Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=warn ts=2026-05-01T23:29:17.476312347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:17.477428725Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:29:20.811176691Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-01T23:29:20.904053527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59474"
level=warn ts=2026-05-01T23:29:22.477118804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:24.144493587Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=debug ts=2026-05-01T23:29:24.144515455Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-01T23:29:24.572857223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59440"
level=info ts=2026-05-01T23:29:27.476503199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:29:27.476533211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:29:27.47654117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:29:27.476563447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:27.477898313Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=debug ts=2026-05-01T23:29:28.655129567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:29:30.810771047Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-01T23:29:31.300225118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:29:31.310937112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:29:32.476363963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:29:33.348925846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:29:34.145277311Z caller=checkpoint.go:372 msg="writing series" size="846 kB"
level=debug ts=2026-05-01T23:29:34.145299575Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-01T23:29:36.955214447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:29:37.476918463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:37.478039369Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:29:40.811625991Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:29:42.476702909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:29:43.832059792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:29:43.837428531Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:29:44.1444552Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-01T23:29:45.560793257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:29:45.564601362Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:29:47.476420127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:47.477826808Z caller=checkpoint.go:372 msg="writing series" size="896 kB"
level=debug ts=2026-05-01T23:29:47.477844708Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-01T23:29:50.811617942Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:29:52.479883401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:29:53.191319808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:29:53.198608651Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:29:53.791161921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:29:53.797769726Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:29:54.14493015Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-01T23:29:57.457504528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:29:57.458549233Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:29:57.477040325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:29:57.477077015Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:29:57.477086344Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:29:57.477101811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:29:57.522259553Z caller=checkpoint.go:372 msg="writing series" size="51 MB"
level=debug ts=2026-05-01T23:29:57.522286948Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=3
level=info ts=2026-05-01T23:29:59.52507855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:29:59.532265862Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:30:00.811216596Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:30:02.477260235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:30:03.350756017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:30:04.144900308Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-01T23:30:04.687391321Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:30:07.350245547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:30:07.358509489Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:30:07.476935465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:07.478078015Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-01T23:30:10.812171227Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=warn ts=2026-05-01T23:30:12.477061606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:14.144715591Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:30:17.476322773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:17.477917873Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=debug ts=2026-05-01T23:30:17.477933066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-01T23:30:20.81184694Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-01T23:30:22.476483419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:24.144626961Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-01T23:30:27.321324647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:30:27.322935325Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T23:30:27.476798284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:30:27.476802907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:27.476823266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:30:27.476833837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:30:27.47791264Z caller=checkpoint.go:384 msg="flushing series" totalSize="322 kB" series=2
level=info ts=2026-05-01T23:30:27.484809504Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-01T23:30:27.486570511Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009263174s
level=info ts=2026-05-01T23:30:29.898235622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:30:29.902200277Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:30:31.485077662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:30:31.488996956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:30:32.476488395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:30:33.090772124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:30:33.352291384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=warn ts=2026-05-01T23:30:37.47640462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:30:41.418670848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:30:41.427761179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:30:42.477101455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:30:45.260569374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:30:45.266474093Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:30:46.378481626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:30:46.385872955Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:30:47.476502933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:30:50.909698666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43712"
level=warn ts=2026-05-01T23:30:52.476209006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:30:54.579320883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49404"
level=info ts=2026-05-01T23:30:55.202095531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:30:55.211821491Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:30:57.457219142Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:30:57.458287729Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:30:57.459559676Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-01T23:30:57.459601001Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975197 n=1
level=debug ts=2026-05-01T23:30:57.459647845Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975197 n=1
level=info ts=2026-05-01T23:30:57.476791623Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:30:57.476794864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:30:57.476811301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:30:57.476819054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:30:57.47684131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:30:57.47688619Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-01T23:30:57.476893634Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:30:57.476910852Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.368µs success=true
level=debug ts=2026-05-01T23:30:57.477117771Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-01T23:30:58.02379721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:31:00.810677716Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-01T23:31:02.476423227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:31:03.353577261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:31:04.144905704Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
ts=2026-05-01T23:31:05.108342173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46308"
level=warn ts=2026-05-01T23:31:07.477098791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:07.477118566Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-01T23:31:10.811527605Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-01T23:31:10.81155576Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-01T23:31:12.222253979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:31:12.228054561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:31:12.228077694Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:31:12.228106167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:31:12.232244227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:31:12.476404853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:14.144292765Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-01T23:31:14.233356978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="49 MB" avg_comp="2.7 MB" total_uncomp="403 MB" avg_uncomp="22 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:31:14.23702084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:31:15.403652862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:31:17.47675531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:17.478075672Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-01T23:31:20.811674044Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-01T23:31:20.911473259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54002"
level=warn ts=2026-05-01T23:31:22.476902381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:24.146095167Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:31:26.320761276Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:31:27.382973011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:31:27.3882959Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:31:27.476484733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:31:27.476510563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:31:27.476510497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:31:27.476519636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:31:27.478147872Z caller=checkpoint.go:372 msg="writing series" size="616 kB"
level=debug ts=2026-05-01T23:31:27.478160355Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-01T23:31:28.06419659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.6 MB" avg_comp="2.2 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:31:28.07341601Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:31:30.810857769Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-01T23:31:31.659374156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:31:31.6685032Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:31:32.476675391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:31:33.354781395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:31:34.143768247Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=warn ts=2026-05-01T23:31:37.476413746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:37.477551561Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-01T23:31:39.453836781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:31:39.46162763Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:31:40.811321746Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-01T23:31:41.237119129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:31:41.243996499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:31:42.476589859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:44.14457873Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=warn ts=2026-05-01T23:31:47.47630145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:47.477401072Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:31:50.811122162Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-01T23:31:51.074991891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:31:51.081260629Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:31:52.477111559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:31:54.144889171Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-01T23:31:57.457060163Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:31:57.458104773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T23:31:57.476716688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:31:57.476727627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:31:57.476743375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:31:57.47675358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:31:57.477876486Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-01T23:31:58.338182817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:31:58.341990666Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:32:00.810887107Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-01T23:32:01.289824246Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:32:01.557331438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:32:02.477098411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:32:03.356612664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:32:04.144365867Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=debug ts=2026-05-01T23:32:04.144388305Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-01T23:32:07.476410146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:07.477570517Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-01T23:32:09.898292056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:32:09.910255138Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:32:10.814053208Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-01T23:32:12.476142103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:14.144840968Z caller=checkpoint.go:372 msg="writing series" size="293 kB"
level=warn ts=2026-05-01T23:32:17.476604111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:17.477736604Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-01T23:32:18.935242975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:32:18.94401236Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:32:18.944039958Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:32:18.944075844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:32:18.952431223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:32:20.812255391Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:32:22.476866404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:24.144450126Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-01T23:32:24.144474211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-01T23:32:25.81005327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:32:25.820435724Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:32:27.476913178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:32:27.476935561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:32:27.476957659Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:32:27.476983878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:27.478096905Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=debug ts=2026-05-01T23:32:29.559755437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:32:30.811068342Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:32:32.477099949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:32:33.358442325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:32:34.144340287Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-01T23:32:35.113397628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48388"
level=info ts=2026-05-01T23:32:35.616678829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:32:35.625445129Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:32:37.476925858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:37.47793269Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-01T23:32:37.593029826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:32:37.614853826Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:32:39.975092071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:32:39.978691132Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:32:39.978709589Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:32:39.978743455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:32:39.981508176Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:32:40.814070395Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:32:42.47689293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:44.144167933Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T23:32:47.477107895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:47.47721339Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-01T23:32:48.977134348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:32:48.981997699Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:32:50.587019532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:32:50.810988853Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=debug ts=2026-05-01T23:32:50.811016106Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-01T23:32:52.476496176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:54.143983503Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T23:32:57.396748428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:32:57.404227494Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:32:57.456892698Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:32:57.457968314Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:32:57.476600954Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:32:57.476627512Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:32:57.476639481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:32:57.476660891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:32:57.477862453Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=info ts=2026-05-01T23:32:58.525895868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:32:58.53491638Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:33:00.810905093Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-01T23:33:02.476984163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:33:03.359418579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:33:04.144890811Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=debug ts=2026-05-01T23:33:04.374657221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:33:05.115449466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36670"
level=info ts=2026-05-01T23:33:05.327538738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:33:05.335745059Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:33:07.476232295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:07.477391053Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-01T23:33:10.81091914Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-01T23:33:12.476812432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:14.1443351Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-05-01T23:33:17.476147541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:17.477458174Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:33:17.477472823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-01T23:33:20.407379233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:33:20.421717396Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:33:20.810924581Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
ts=2026-05-01T23:33:20.91978448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43938"
level=info ts=2026-05-01T23:33:21.588921154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:33:21.596089562Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:33:22.47636146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:33:23.09378042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:33:23.098109831Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:33:24.144898923Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-01T23:33:27.476220668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:33:27.47624545Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:33:27.476253183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:33:27.476298453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:27.487025665Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:33:27.487054975Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=debug ts=2026-05-01T23:33:30.811185448Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:33:32.476590854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:33:33.360417754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:33:33.810842193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:33:34.154358303Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:33:34.154382114Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
ts=2026-05-01T23:33:35.117001293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53648"
level=debug ts=2026-05-01T23:33:35.295649966Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:33:37.477122717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:37.477142485Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-01T23:33:39.887562047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:33:39.898843083Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:33:40.811280535Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-01T23:33:42.477193887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:44.144998061Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=warn ts=2026-05-01T23:33:47.476654426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:47.479630074Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-01T23:33:47.479645325Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
level=info ts=2026-05-01T23:33:47.68000459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:33:47.689768105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:33:48.402348161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:33:48.407294316Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:33:50.815033274Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-01T23:33:50.815057844Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=1
ts=2026-05-01T23:33:50.921259771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34530"
level=warn ts=2026-05-01T23:33:52.476802642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:54.145170528Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-01T23:33:57.457812183Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:33:57.458292077Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:33:57.476781881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:33:57.476796372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:33:57.47680354Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:33:57.476814848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:33:57.479352072Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-01T23:33:57.47936521Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
level=debug ts=2026-05-01T23:34:00.811379848Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=debug ts=2026-05-01T23:34:01.564286048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:34:02.109665367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:34:02.121366226Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-01T23:34:02.121389214Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:34:02.12141432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:34:02.12898071Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:34:02.477139819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:34:03.362422383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:34:04.144891897Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
ts=2026-05-01T23:34:05.118523885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47022"
level=info ts=2026-05-01T23:34:06.539671741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:34:06.543194665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:34:07.476252207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:07.477309269Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:34:10.510238761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:34:10.517532689Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:34:10.517558052Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:34:10.51758724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:34:10.524049175Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:34:10.818477445Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-01T23:34:10.818505194Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.9 MB" series=4
level=warn ts=2026-05-01T23:34:12.476996246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:14.145409105Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=warn ts=2026-05-01T23:34:17.476737371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:17.477853639Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:34:19.998154048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:34:20.005341169Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:34:20.810615837Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-01T23:34:20.92299198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49864"
level=warn ts=2026-05-01T23:34:22.476314941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:34:22.845671059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:34:22.852917135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:34:24.144525946Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-01T23:34:24.144551725Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-01T23:34:27.476930206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:34:27.476975694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:34:27.476969557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:34:27.476983313Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:34:27.478292653Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-01T23:34:28.577149204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:34:28.584584281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:34:28.998629057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:34:29.000310039Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:34:30.434870904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:34:30.434929295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="802 kB" avg_comp="802 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:34:30.43943982Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:34:30.810632107Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:34:32.476509947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:34:33.222733913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:34:33.232227099Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-01T23:34:33.364039174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:34:34.144377724Z caller=checkpoint.go:372 msg="writing series" size="889 kB"
level=debug ts=2026-05-01T23:34:34.144402963Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-01T23:34:35.120096196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58526"
level=warn ts=2026-05-01T23:34:37.476985621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:37.478141206Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:34:40.811060346Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-01T23:34:41.910952751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:34:41.915134772Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:34:42.476978585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:34:44.132431563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:34:44.138192608Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:34:44.144408589Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-01T23:34:47.476792542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:47.47851028Z caller=checkpoint.go:372 msg="writing series" size="914 kB"
level=debug ts=2026-05-01T23:34:47.478528051Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-01T23:34:50.810995963Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-01T23:34:50.924525513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44486"
level=warn ts=2026-05-01T23:34:52.477140255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:34:52.637473819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:34:52.641340953Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:34:54.144069556Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-01T23:34:57.256348741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:34:57.263638195Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:34:57.339979441Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:34:57.457491938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:34:57.458557127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:34:57.477032946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:34:57.477058442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:34:57.477065808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:34:57.477064996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:34:57.526771042Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-01T23:34:57.52679777Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=3
level=info ts=2026-05-01T23:34:58.121707609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:34:58.12934898Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:35:00.811623327Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:35:02.476652462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:35:03.365051594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:35:04.144164571Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-01T23:35:07.477075358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:35:07.477158791Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:35:10.811108303Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=info ts=2026-05-01T23:35:11.088401242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:35:11.095550997Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-01T23:35:12.476708633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:35:14.144583874Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:35:17.476210587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:35:17.477868241Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=debug ts=2026-05-01T23:35:17.477886866Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-01T23:35:17.867917032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:35:17.87609026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:35:20.81109039Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-01T23:35:22.476316155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:35:24.143839136Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-01T23:35:25.451599084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:35:27.476766258Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:35:27.476799652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:35:27.476807477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:35:27.476818602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:35:27.477860212Z caller=checkpoint.go:384 msg="flushing series" totalSize="343 kB" series=2
level=info ts=2026-05-01T23:35:27.483734369Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-01T23:35:27.485592179Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008546196s
level=info ts=2026-05-01T23:35:27.689983781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:35:27.693414002Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:35:27.923114503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:35:27.931396876Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:35:27.931425161Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:35:27.931454317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:35:27.939102681Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:35:31.510988752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:35:32.477146236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:35:33.366205876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:35:35.123265735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56042"
level=warn ts=2026-05-01T23:35:37.476229078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:35:38.232079921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:35:38.243878438Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:35:41.62600155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="52 MB" avg_comp="2.7 MB" total_uncomp="427 MB" avg_uncomp="22 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:35:41.629649039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:35:42.015746481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:35:42.02299928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:35:42.476710979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:35:46.030201761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:35:46.036820911Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:35:47.476884261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:35:50.928655835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39400"
level=warn ts=2026-05-01T23:35:52.476593615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:35:54.601381682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42912"
level=info ts=2026-05-01T23:35:57.45762155Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:35:57.457885315Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:35:57.476437997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:35:57.476452638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:35:57.476460452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:35:57.476467892Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-01T23:35:57.476499485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:35:57.476578395Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-01T23:35:57.476584912Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:35:57.476600993Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.799µs success=true
level=debug ts=2026-05-01T23:35:57.476816288Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-01T23:35:58.207057324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:35:58.213679662Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:36:00.811047857Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-01T23:36:02.476972124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:36:03.367924201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:36:03.694704078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:36:04.144557712Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-01T23:36:04.753319457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:36:04.760107909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:36:07.476218693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:07.477311481Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-01T23:36:10.544318419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.6 MB" avg_comp="2.4 MB" total_uncomp="61 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:36:10.550762318Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:36:10.810856553Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-01T23:36:10.810878836Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:36:12.476515775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:13.056962983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:36:14.143907291Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-01T23:36:15.270228766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:36:15.273965702Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:36:17.476858314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:17.47707625Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-01T23:36:20.810427751Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=warn ts=2026-05-01T23:36:22.476300653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:24.143983793Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-01T23:36:24.602967917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34748"
level=info ts=2026-05-01T23:36:24.734544136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:36:24.738902702Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:36:27.476863083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:36:27.476869429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:36:27.476897221Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:36:27.476970237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:36:27.477328372Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-01T23:36:27.477348281Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-01T23:36:30.81081173Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-01T23:36:32.476455428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:33.16725412Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:36:33.368926698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:36:34.144085075Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=info ts=2026-05-01T23:36:34.548651725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:36:34.55820998Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
ts=2026-05-01T23:36:35.127309051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46818"
level=info ts=2026-05-01T23:36:35.423358501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:36:35.430612589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:36:37.476869041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:37.476874121Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-01T23:36:40.810205696Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-01T23:36:42.476840759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:44.144077429Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=warn ts=2026-05-01T23:36:47.476736541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:47.476846134Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:36:50.810823467Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:36:52.476414478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:36:54.143786629Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-01T23:36:54.624150917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:36:54.629824419Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-01T23:36:57.274271851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:36:57.281083766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:36:57.456801476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:36:57.457874516Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:36:57.476415815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:36:57.476443046Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:36:57.476440302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:36:57.476453448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:36:57.477575494Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:37:00.811169825Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T23:37:02.476815465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:03.270590062Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:37:03.270640931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="80 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:37:03.278060991Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-01T23:37:03.370917221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:37:04.144606796Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
level=debug ts=2026-05-01T23:37:04.144629581Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=info ts=2026-05-01T23:37:05.872828316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:37:05.877166784Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:37:06.304227915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:37:06.307675006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:37:07.476959138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:07.47710832Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-01T23:37:10.81069234Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-01T23:37:12.47700038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:13.690586149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:37:14.14417569Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=warn ts=2026-05-01T23:37:17.477169525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:17.477263718Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-01T23:37:20.811033066Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-01T23:37:20.933930921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38938"
level=warn ts=2026-05-01T23:37:22.476405344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:24.144027666Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-01T23:37:24.144047787Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-01T23:37:27.476440201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:37:27.476452645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:27.476471545Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:37:27.476480341Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:37:27.477604903Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:37:28.927695857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:37:28.935452184Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:37:29.583384961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:37:29.591625966Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:37:30.568433377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:37:30.811015573Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-01T23:37:32.476378447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:37:33.372789045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:37:34.1437349Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-01T23:37:35.130614929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48766"
level=warn ts=2026-05-01T23:37:37.476689265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:37.477782512Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-01T23:37:40.810482989Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:37:42.476301814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:44.143874761Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T23:37:47.477151159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:37:47.477298906Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-01T23:37:48.819967123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:37:48.825441106Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-01T23:37:49.717927692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="80 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:37:49.726530204Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:37:50.810675473Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-01T23:37:50.810700499Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
ts=2026-05-01T23:37:50.935472711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39972"
level=warn ts=2026-05-01T23:37:52.476147081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:37:53.983156143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:37:53.986695952Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:37:54.143578989Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
ts=2026-05-01T23:37:54.609124194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43812"
level=debug ts=2026-05-01T23:37:55.040513617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:37:56.484131696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:37:56.493106965Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:37:57.457394724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:37:57.458432539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:37:57.476972132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:37:57.476994234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-01T23:37:57.476988235Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=warn ts=2026-05-01T23:37:57.476991637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:37:57.477002372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:37:58.286265464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:38:00.810970156Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-01T23:38:02.476416308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:38:03.143707013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:38:03.149763874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:38:03.149786822Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:38:03.149815455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:38:03.153351531Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-01T23:38:03.374059904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:38:04.143902254Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
ts=2026-05-01T23:38:05.132050487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43706"
level=warn ts=2026-05-01T23:38:07.47658384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:07.477605964Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-01T23:38:10.810702919Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-01T23:38:12.477145084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:14.144081727Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-01T23:38:17.476978726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:17.477156937Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:38:17.477179843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-05-01T23:38:20.810965812Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-01T23:38:21.94983558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:38:21.954836332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:38:22.476994619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:24.143696058Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-05-01T23:38:24.612878298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46834"
level=debug ts=2026-05-01T23:38:26.505172205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:38:27.476979095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:38:27.477005916Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:38:27.477036076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:38:27.477046044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:38:27.497208542Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-01T23:38:27.497240164Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-05-01T23:38:27.747218862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:38:27.754763213Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:38:29.19702152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:38:29.215217283Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:38:30.813115105Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:38:32.476839625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:38:33.37526513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:38:34.158670917Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-01T23:38:34.158703173Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
ts=2026-05-01T23:38:35.133136845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57170"
level=warn ts=2026-05-01T23:38:37.476632856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:37.477632008Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-01T23:38:37.920176975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:38:37.925830045Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-01T23:38:38.417674224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:38:38.426125878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:38:38.426146806Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:38:38.426173722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:38:38.433605118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:38:40.810464035Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-01T23:38:41.424212057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:38:41.426272373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T23:38:42.477006553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:38:42.885600164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="80 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:38:42.898057891Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:38:44.14401663Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=warn ts=2026-05-01T23:38:47.476273496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:47.478409521Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-01T23:38:47.478427539Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-05-01T23:38:49.974151079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:38:49.983645096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:38:50.816030164Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-01T23:38:50.816061546Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=1
level=info ts=2026-05-01T23:38:51.603665487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:38:51.612830551Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:38:51.612858801Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:38:51.612899073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:38:51.616297307Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:38:52.476776692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:38:54.144054626Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
ts=2026-05-01T23:38:54.615106398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44970"
level=info ts=2026-05-01T23:38:54.84399619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:38:54.847732624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:38:57.457134985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:38:57.458184728Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T23:38:57.476690929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:38:57.47668907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:38:57.476707837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:38:57.47671794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:38:57.477634128Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:38:57.477646671Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=debug ts=2026-05-01T23:39:00.81170294Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=debug ts=2026-05-01T23:39:01.262656389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:39:02.476733285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:39:03.376563762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:39:04.143758286Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
ts=2026-05-01T23:39:05.135044156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55336"
level=info ts=2026-05-01T23:39:05.20905199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:39:05.217246592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:39:07.47629866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:07.477411988Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-01T23:39:10.819175338Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:39:10.819197445Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-05-01T23:39:12.369140599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:39:12.37757137Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:39:12.477138934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:14.145056212Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-01T23:39:14.786205238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:39:14.796048079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:39:17.476561327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:17.47766123Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:39:20.810270539Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=info ts=2026-05-01T23:39:22.416288832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:39:22.424191229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:39:22.476506292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:24.14369881Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-01T23:39:24.580062684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="80 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:39:24.592314708Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:39:27.476496342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:39:27.47651191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:27.476525194Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:39:27.476533964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:39:27.478060145Z caller=checkpoint.go:372 msg="writing series" size="594 kB"
level=debug ts=2026-05-01T23:39:27.478074099Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-01T23:39:30.632056628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:39:30.637407116Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:39:30.811090947Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:39:32.476677084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:33.056577715Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:39:33.377496404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:39:34.143778375Z caller=checkpoint.go:372 msg="writing series" size="844 kB"
level=info ts=2026-05-01T23:39:36.167191683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:39:36.170725101Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:39:37.476535545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:37.477700086Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:39:37.612427831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:39:40.814090424Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-01T23:39:42.476734757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:39:43.266840058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:39:43.271080169Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:39:44.143887922Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-01T23:39:44.143911154Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-01T23:39:47.47697986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:47.477313933Z caller=checkpoint.go:372 msg="writing series" size="931 kB"
level=debug ts=2026-05-01T23:39:50.811257135Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:39:52.477074565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:54.144341577Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-01T23:39:57.457970223Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:39:57.457990861Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:39:57.476515714Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:39:57.476537496Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:39:57.476547045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:39:57.476542154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:39:57.540004505Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-01T23:39:57.540033483Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=info ts=2026-05-01T23:39:57.967520909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:39:57.977683596Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:39:59.018146583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:39:59.026338105Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:40:00.810544835Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:40:02.477093509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:40:03.379051153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:40:04.144534908Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-01T23:40:06.604163924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:40:07.477229626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:40:07.477287222Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-01T23:40:08.160977417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:40:08.169906839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:40:09.410627676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="80 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:40:09.420289248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:40:10.811738554Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=warn ts=2026-05-01T23:40:12.477014895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:40:14.14425668Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:40:17.476757546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:40:17.47724004Z caller=checkpoint.go:372 msg="writing series" size="472 kB"
level=debug ts=2026-05-01T23:40:17.477257407Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-01T23:40:20.811686287Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-01T23:40:21.913497968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:40:21.921029187Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:40:22.476797021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:40:23.733178547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:40:23.738116807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:40:24.144416968Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-01T23:40:27.476269839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:40:27.476304567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:40:27.476316808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:40:27.476318383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:40:27.477380868Z caller=checkpoint.go:384 msg="flushing series" totalSize="368 kB" series=2
level=info ts=2026-05-01T23:40:27.48447848Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-01T23:40:27.486441988Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00970309s
level=info ts=2026-05-01T23:40:28.501027527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:40:28.51699314Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-01T23:40:32.476412489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:40:33.057205617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:40:33.380521013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:40:37.213113796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:40:37.476970705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:40:40.304412405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:40:40.315756036Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-01T23:40:40.315783606Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:40:40.315823559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:40:40.323326529Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:40:42.476991557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:40:47.47626862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:40:52.476153811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:40:54.148324081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:40:54.158241314Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:40:55.359357729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:40:55.367734123Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:40:57.457365542Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:40:57.458427037Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:40:57.476910064Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:40:57.476921537Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:40:57.476959867Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:40:57.476971689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:40:57.477001344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:40:57.477078201Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-01T23:40:57.477088759Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:40:57.477113566Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.809µs success=true
level=debug ts=2026-05-01T23:40:57.477396358Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-01T23:41:00.811556883Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-01T23:41:01.456886805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:41:01.464927971Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:41:01.464976846Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:41:01.465008308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:41:01.471320269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:41:01.928583377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:41:01.936196269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:41:02.476458217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:03.155911478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:41:03.381716579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:41:04.145329818Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=warn ts=2026-05-01T23:41:07.476285794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:07.477385992Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-01T23:41:09.863955887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:41:09.868219099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:41:10.811572859Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-01T23:41:10.8116007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:41:12.47701228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:14.144564324Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-01T23:41:15.114217386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:41:15.122274247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:41:17.476215576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:17.478808906Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-01T23:41:17.87829017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:41:17.882305359Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:41:20.811965417Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=debug ts=2026-05-01T23:41:22.215347001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:41:22.476531636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:41:22.663379808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:41:22.671282103Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:41:24.144667777Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:41:27.476466342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:41:27.476492459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:41:27.476500462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:41:27.476496605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:27.478022911Z caller=checkpoint.go:372 msg="writing series" size="890 kB"
level=debug ts=2026-05-01T23:41:27.478034031Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=debug ts=2026-05-01T23:41:27.654570919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:41:27.923996238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:41:27.928316697Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-01T23:41:27.928338939Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:41:27.928360427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:41:27.932028011Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:41:30.810840954Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-01T23:41:32.477057922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:41:33.383806321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:41:34.144049253Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=info ts=2026-05-01T23:41:36.475685405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:41:36.483475725Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:41:37.476584429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:37.477798009Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=debug ts=2026-05-01T23:41:40.811478481Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-01T23:41:42.476447503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:44.144903847Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=info ts=2026-05-01T23:41:46.568765361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:41:46.572923846Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-01T23:41:47.476141995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:47.478287088Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-01T23:41:47.647534518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:41:47.654987851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:41:50.810859383Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-01T23:41:52.476742195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:53.583096322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:41:53.583159755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:41:53.590231253Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:41:53.879102823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:41:53.888548221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:41:53.888576672Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:41:53.888601003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:41:53.89518161Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:41:54.145105083Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-01T23:41:57.283385334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="55 MB" avg_comp="2.8 MB" total_uncomp="451 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:41:57.287678049Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:41:57.457219521Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:41:57.458264848Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:41:57.476771084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:41:57.476780061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:41:57.476788634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:41:57.476796633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:41:57.477901689Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:42:00.811202109Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T23:42:02.476731765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:42:03.384978429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:42:04.145231067Z caller=checkpoint.go:372 msg="writing series" size="389 kB"
level=debug ts=2026-05-01T23:42:04.145253338Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-01T23:42:04.466231011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:42:04.474675029Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-01T23:42:05.146426946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43120"
level=info ts=2026-05-01T23:42:05.954340858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:42:05.959561839Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:42:06.957759303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:42:06.965543477Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:42:07.476372617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:07.477666505Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-01T23:42:08.324926776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:42:08.329393083Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:42:09.068689464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:42:10.813168959Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-01T23:42:11.432783044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:42:11.43670246Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:42:12.477038356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:14.144294151Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=warn ts=2026-05-01T23:42:17.477061268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:17.478286012Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-01T23:42:20.811198105Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:42:22.476608665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:24.14421009Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-01T23:42:24.144233058Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-01T23:42:24.467357845Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:42:24.633063073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37914"
level=info ts=2026-05-01T23:42:27.476423346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:42:27.476451269Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:42:27.476459586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:42:27.476467006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:27.477568884Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:42:27.686318946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:42:27.700125976Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:42:30.811124007Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-01T23:42:32.23699831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:42:32.242424858Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:42:32.47689584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:42:33.386144535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:42:34.144034807Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-01T23:42:36.442846471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:42:36.450266596Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:42:37.476309332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:37.477430882Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-01T23:42:38.060645147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:42:38.064831568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:42:40.152066745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:42:40.159121473Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:42:40.811100309Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:42:42.476496993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:44.14502696Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-01T23:42:47.476982836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:47.478242133Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-01T23:42:49.227211551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:42:49.229083373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:42:50.812022462Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-01T23:42:50.812061036Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-05-01T23:42:51.464341507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:42:52.476200075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:42:54.144443711Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-01T23:42:57.456653263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:42:57.457692494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:42:57.476239558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:42:57.476256445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:42:57.476259078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:42:57.476264529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:42:57.477362305Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=info ts=2026-05-01T23:42:58.469250251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:42:58.476575809Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:43:00.811377902Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-01T23:43:02.476474209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:43:03.387522889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:43:03.898879245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:43:03.90352119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:43:04.144785987Z caller=checkpoint.go:372 msg="writing series" size="548 kB"
level=warn ts=2026-05-01T23:43:07.47708596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:07.478166399Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-01T23:43:09.940747344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:43:09.945412644Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:43:10.810802988Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=warn ts=2026-05-01T23:43:12.477048509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:43:13.056816431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:43:13.062423513Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:43:14.144189743Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-01T23:43:14.144222371Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-01T23:43:17.47685398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:17.478375753Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-01T23:43:20.505378426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:43:20.513803804Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:43:20.811612901Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
ts=2026-05-01T23:43:20.956263197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34416"
level=info ts=2026-05-01T23:43:21.20911502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:43:21.21621598Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:43:22.477082041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:24.144319175Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-01T23:43:25.210969441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:43:25.319400093Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:43:26.534771053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:43:26.543512576Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:43:27.476973447Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:43:27.477005516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:43:27.476999409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:43:27.47701725Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:43:27.493673023Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-01T23:43:27.493701922Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=4
level=debug ts=2026-05-01T23:43:30.811313191Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-01T23:43:32.054554529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:43:32.058150963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:43:32.477020459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:43:33.389851514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:43:34.160830805Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-01T23:43:34.160856278Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=warn ts=2026-05-01T23:43:37.476402381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:37.477523666Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-01T23:43:40.811671406Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-01T23:43:42.476486906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:43:43.175201563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:43:43.18678426Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:43:44.144524056Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=warn ts=2026-05-01T23:43:47.477106941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:47.480320273Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-01T23:43:47.480334772Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=debug ts=2026-05-01T23:43:50.815544485Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-05-01T23:43:50.815572403Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=1
ts=2026-05-01T23:43:50.95747502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:36090"
level=warn ts=2026-05-01T23:43:52.476336679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:54.144681898Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-01T23:43:54.221236065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:43:54.226954416Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:43:54.858136159Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:43:57.457639595Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:43:57.457644334Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:43:57.476100207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:43:57.476118183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:43:57.476127673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:43:57.476137477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:43:57.480623559Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-01T23:43:57.480636396Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=debug ts=2026-05-01T23:44:00.811107931Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-01T23:44:02.477087557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:44:02.960708956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:44:02.966333966Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-01T23:44:03.201458627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:44:03.207383201Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-01T23:44:03.391193862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:44:04.144178258Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-01T23:44:07.223594896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:44:07.230812281Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:44:07.476658757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:07.477767417Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:44:08.248887614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:44:08.256609118Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:44:10.814661857Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:44:10.81468608Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=debug ts=2026-05-01T23:44:12.297182612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:44:12.477218833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:14.145089161Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=warn ts=2026-05-01T23:44:17.47683883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:17.477935984Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:44:20.811606966Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:44:22.477097886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:24.145438338Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-01T23:44:24.145463738Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-01T23:44:25.107568688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:44:25.116215261Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:44:25.429011394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:44:25.437150205Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:44:25.448367583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:44:25.451479258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:44:27.476959712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:44:27.476971223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:27.476986963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:44:27.47699735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:44:27.478375194Z caller=checkpoint.go:372 msg="writing series" size="595 kB"
level=debug ts=2026-05-01T23:44:27.706272922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:44:30.811399962Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-01T23:44:32.476777353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:44:33.392395813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:44:34.145001053Z caller=checkpoint.go:372 msg="writing series" size="762 kB"
level=debug ts=2026-05-01T23:44:34.14502397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-01T23:44:37.476423404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:37.477598693Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-01T23:44:40.811441859Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-01T23:44:41.600169751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:44:41.6049049Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:44:41.604925433Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:44:41.6049678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:44:41.607981243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:44:42.476390717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:44.145141639Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=warn ts=2026-05-01T23:44:47.476641834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:47.479238083Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:44:47.479254265Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=info ts=2026-05-01T23:44:49.254338767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:44:49.260054819Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:44:50.810785999Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-01T23:44:52.046629514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:44:52.054921361Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:44:52.476547054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:54.069302747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:44:54.144536923Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-01T23:44:54.265053864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-01T23:44:54.643283748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39066"
level=info ts=2026-05-01T23:44:57.457606394Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:44:57.457629514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:44:57.477128268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:44:57.477135445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:44:57.477147799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:44:57.47715678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:44:57.536026696Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-01T23:44:57.536057821Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-01T23:44:57.771510708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:44:57.77576726Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:44:59.146073267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:44:59.150596362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:45:00.811285643Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-01T23:45:00.817008713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:45:00.826067987Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:45:02.476885013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:45:03.394235855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:45:04.144394241Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-01T23:45:07.476926602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:45:07.478144421Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-01T23:45:10.812415431Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=info ts=2026-05-01T23:45:11.599383542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:45:11.606551004Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:45:11.606575807Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:45:11.606611644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:45:11.609558206Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:45:12.477017503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:45:14.144845178Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-01T23:45:17.477116447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:45:17.478545585Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
level=debug ts=2026-05-01T23:45:17.478562106Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-01T23:45:19.221255259Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:45:20.810880557Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-01T23:45:22.092973879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:45:22.103526786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:45:22.103553068Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:45:22.103581579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:45:22.110844581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:45:22.476248762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:45:23.163502944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:45:23.171591049Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:45:24.144604945Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-01T23:45:24.784168039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:45:24.7875988Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:45:27.476888682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:45:27.476903179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:45:27.476914969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:45:27.476923921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:45:27.478001756Z caller=checkpoint.go:384 msg="flushing series" totalSize="377 kB" series=2
level=info ts=2026-05-01T23:45:27.483826603Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-01T23:45:27.485624002Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008335035s
level=info ts=2026-05-01T23:45:29.985240678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:45:29.988922705Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:45:30.954616886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:45:32.476670608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:45:33.396336223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:45:35.275144143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:45:35.2832353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:45:37.053124474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:45:37.059428532Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:45:37.47673778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:45:37.979022207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:45:37.997421278Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:45:42.476197312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:45:43.889170438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:45:43.899525117Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:45:46.640290563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:45:46.644799837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-01T23:45:47.477028645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:45:52.476996253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:45:54.583371682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:45:57.457852084Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:45:57.458278431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-01T23:45:57.4595308Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-01T23:45:57.459576465Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975198 n=1
level=debug ts=2026-05-01T23:45:57.459617151Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975198 n=1
level=info ts=2026-05-01T23:45:57.476659601Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:45:57.476662548Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:45:57.47667841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:45:57.476688388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:45:57.476689422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:45:57.476757932Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-01T23:45:57.476765291Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:45:57.476782201Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.529µs success=true
level=debug ts=2026-05-01T23:45:57.476987348Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-01T23:46:00.810589148Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-01T23:46:02.476490845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:46:02.547771585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:46:02.556291632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-01T23:46:03.398074639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-01T23:46:03.646637168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:46:03.650428299Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:46:04.144242624Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=warn ts=2026-05-01T23:46:07.477063729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:07.477105426Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-01T23:46:10.021085714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:46:10.02523421Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:46:10.811693528Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=debug ts=2026-05-01T23:46:10.811722039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:46:12.476517059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:46:13.733600391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:46:13.741197992Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:46:14.14442651Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-01T23:46:17.477039236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:17.477386384Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=debug ts=2026-05-01T23:46:20.811471837Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-01T23:46:20.965667033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52600"
level=info ts=2026-05-01T23:46:21.423293862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:46:21.430389516Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:46:22.476510778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:24.144461551Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-01T23:46:24.372760143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:46:24.374475741Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:46:24.649899493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39048"
level=info ts=2026-05-01T23:46:27.4765108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:46:27.476534916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:46:27.476542902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:46:27.476560611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:27.477983857Z caller=checkpoint.go:372 msg="writing series" size="810 kB"
level=debug ts=2026-05-01T23:46:27.478000506Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=debug ts=2026-05-01T23:46:29.164172789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:46:29.164228801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:46:29.167485918Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:46:30.682345561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:46:30.689277936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:46:30.810663878Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-01T23:46:32.476964002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:46:33.39969656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:46:34.143934061Z caller=checkpoint.go:372 msg="writing series" size="814 B"
level=info ts=2026-05-01T23:46:37.000690832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:46:37.008311275Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:46:37.47650032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:37.477619592Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-01T23:46:40.811239224Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-01T23:46:42.476347814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:46:42.585692336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:46:42.58954216Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:46:43.460552104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:46:43.464886746Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:46:44.143749304Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-01T23:46:47.476487358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:47.477622942Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:46:50.811026055Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-01T23:46:52.477128825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:54.144223233Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-01T23:46:55.41737304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:46:55.424302088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:46:57.457077809Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:46:57.4581403Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:46:57.476517038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:46:57.476526623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:46:57.47653418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:46:57.476542328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:46:57.477668135Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-01T23:46:59.071594395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:46:59.078901767Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:46:59.739682204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:46:59.746424199Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-01T23:46:59.746443246Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:46:59.746466177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:46:59.752577342Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:47:00.810557574Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-01T23:47:00.892077274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:47:00.892106899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-01T23:47:00.896525676Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:47:01.192106977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:47:01.195247815Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:47:01.492298448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:47:01.495253962Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:47:01.792936618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="142 kB" avg_uncomp="142 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:47:01.796105928Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:47:02.092990966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:47:02.093276258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:47:02.09659214Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:47:02.096596852Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:47:02.392440554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:47:02.395528135Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-01T23:47:02.476730169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:47:02.692145689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:47:02.695082517Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-01T23:47:03.40147981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:47:04.143872775Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=debug ts=2026-05-01T23:47:04.143898434Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
ts=2026-05-01T23:47:05.165074464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53704"
level=warn ts=2026-05-01T23:47:07.476670527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:07.478093449Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=debug ts=2026-05-01T23:47:10.811258872Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-01T23:47:11.171437812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:47:11.934496119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:47:11.937228488Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:47:12.208876135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:47:12.215422537Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:47:12.215445944Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:47:12.215472015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:47:12.220600747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:47:12.476550371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:14.144248873Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=info ts=2026-05-01T23:47:14.236304958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:47:14.239871036Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:47:14.922911371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:47:14.926281775Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:47:16.118477277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:47:16.126269319Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:47:16.559569351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:47:16.562781522Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:47:16.65211176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:47:16.655084992Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:47:17.477172962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:17.477334655Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-01T23:47:18.1450817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:47:18.15021184Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:47:18.729916816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:47:18.732996305Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:47:20.709839354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:47:20.71167841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:47:20.713393963Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:47:20.713393151Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:47:20.810871369Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-01T23:47:20.81089039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-01T23:47:22.476577537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:24.143792711Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
ts=2026-05-01T23:47:24.654220815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44372"
level=warn ts=2026-05-01T23:47:27.476727004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:47:27.476734816Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:47:27.476752594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:47:27.476760307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:47:27.477886292Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:47:28.432064612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:47:28.436752202Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-01T23:47:29.381885336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:47:29.381934631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="97 kB" avg_uncomp="97 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:47:29.385631615Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:47:30.811003342Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-01T23:47:31.149758349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:47:31.157407788Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:47:32.476417236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:47:33.402706417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:47:33.461765079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:47:33.470515704Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:47:34.144354177Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-01T23:47:35.835070195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:47:35.8380509Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:47:36.76413976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:47:36.767878692Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:47:37.476352379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:37.477500431Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-01T23:47:40.810330798Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-01T23:47:41.445572745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:47:41.452628382Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:47:42.4770695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:47:43.280066979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:47:43.283216082Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:47:44.143703409Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-01T23:47:44.988080851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:47:44.991553707Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:47:45.350574277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:47:45.353611391Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:47:45.900127623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:47:45.903028733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:47:46.151783585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:47:46.155079463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:47:47.476776572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:47.477953764Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-01T23:47:47.796448479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:47:47.803492279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:47:49.237362438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:47:50.811437152Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-01T23:47:50.811462499Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-05-01T23:47:52.15656287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:47:52.159504597Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:47:52.467444024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:47:52.470116892Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-01T23:47:52.477179125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:54.1438403Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-01T23:47:54.298290969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:47:54.301328343Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-01T23:47:57.457358968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:47:57.458406392Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:47:57.476785564Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:47:57.476791564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:47:57.476801874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:47:57.476809941Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:47:57.477994713Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-01T23:48:00.810609796Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-01T23:48:02.291141369Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:48:02.476510116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:48:03.404628868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:48:04.144671362Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=info ts=2026-05-01T23:48:04.258886495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:48:04.266634872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:48:04.266654999Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:48:04.266690648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:48:04.271798858Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-01T23:48:05.168494992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47678"
level=warn ts=2026-05-01T23:48:07.47682157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:07.477883925Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-01T23:48:07.746993028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:48:07.754852594Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:48:08.888118635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="824 kB" avg_comp="824 kB" total_uncomp="6.6 MB" avg_uncomp="6.6 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-01T23:48:08.892856573Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19de5b8d729%3A19de5e3f5eb%3Ae68afcd4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-01T23:48:08.892885487Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:48:08.892917655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:48:08.895678221Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:48:10.738999771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:48:10.742902645Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-01T23:48:10.811141481Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-01T23:48:12.477032351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:14.144161682Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-01T23:48:14.144187055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-01T23:48:17.476223942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:17.477471544Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-01T23:48:19.614564088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:48:19.617809523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:48:20.810539193Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-01T23:48:21.658284184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:48:21.661624427Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-01T23:48:22.476385479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:48:23.113607511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:48:23.117506214Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:48:23.293081084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:48:23.298718847Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:48:24.144388751Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-01T23:48:27.476820889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:48:27.476851732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:48:27.476861189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:48:27.476913276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:27.493719618Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-01T23:48:27.493748243Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=info ts=2026-05-01T23:48:27.652750735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:48:27.65917888Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:48:29.665542957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:48:30.810736323Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-01T23:48:32.21330229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:48:32.476594503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:48:33.24497255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:48:33.251487756Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:48:33.251509614Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:48:33.251537447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:48:33.257511319Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-01T23:48:33.406060755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:48:33.558463427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:48:33.561654301Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:48:33.593724626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:48:33.596342174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:48:34.161803207Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-01T23:48:34.161826144Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=2
level=info ts=2026-05-01T23:48:35.866214045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:48:35.877180042Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:48:36.376495734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:48:36.379388728Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-01T23:48:37.477042055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:37.477122117Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-01T23:48:37.641828482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:48:37.644545111Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:48:39.217357783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:48:39.220572179Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-01T23:48:40.81059965Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-01T23:48:41.032167581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:48:41.035538319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-01T23:48:42.476803167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:48:42.817204103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:48:42.820370231Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:48:44.144279909Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=info ts=2026-05-01T23:48:44.951708688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:48:44.95455609Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-01T23:48:45.473067445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:48:45.478063292Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:48:46.075668905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:48:46.083179128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:48:47.477110672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:47.478962179Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-01T23:48:47.478978017Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=debug ts=2026-05-01T23:48:50.815500654Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-01T23:48:50.815527628Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.8 MB" series=1
level=warn ts=2026-05-01T23:48:52.476491433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:54.145311733Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-01T23:48:54.825486054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:48:54.829273555Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:48:57.456811458Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:48:57.457872425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:48:57.476331948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:48:57.476350898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:48:57.476359237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:48:57.476387775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:48:57.479866165Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-01T23:48:57.479883805Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=debug ts=2026-05-01T23:49:00.811582124Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=debug ts=2026-05-01T23:49:00.899546573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:49:02.476420469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:49:03.4080863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:49:04.144308818Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-01T23:49:06.615251278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:49:06.624694978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:49:07.476150063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:07.477236796Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:49:08.691700338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:49:08.695256256Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-01T23:49:10.819556866Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:49:10.819591373Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-01T23:49:12.477073582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:49:13.393955685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:49:13.397669863Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:49:14.144380664Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-01T23:49:16.747957288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:49:16.750894115Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-01T23:49:17.476308421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:17.477428488Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:49:18.882924127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:49:18.886153301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:49:19.076572136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:49:20.018096123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:49:20.021104663Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:49:20.810987683Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=info ts=2026-05-01T23:49:22.033398544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:49:22.037120726Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:49:22.476359932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:49:23.222597219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:49:23.230305421Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:49:23.554426436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:49:23.557296763Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-01T23:49:24.143963573Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-01T23:49:24.143988515Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-01T23:49:24.809584478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:49:24.812515576Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:49:25.363116845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:49:25.36970982Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:49:25.6878348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:49:25.692449652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:49:27.476445791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:49:27.476477483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:49:27.47648705Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:49:27.476483307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:27.478057163Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=info ts=2026-05-01T23:49:27.651179201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:49:27.654542606Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:49:29.144329534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:49:30.810827741Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-01T23:49:31.07968057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:49:31.087978364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:49:32.476313357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:49:33.409091115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:49:34.144702321Z caller=checkpoint.go:372 msg="writing series" size="777 kB"
level=debug ts=2026-05-01T23:49:34.144726071Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-01T23:49:36.424745553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:49:36.428400467Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:49:37.47702314Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-01T23:49:37.477031728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:49:40.312155494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:49:40.315375949Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-01T23:49:40.810804416Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-01T23:49:41.832819309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:49:41.839214501Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:49:42.477179918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:44.144258646Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=warn ts=2026-05-01T23:49:47.476899061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:47.47849526Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:49:47.478515299Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
level=debug ts=2026-05-01T23:49:50.810651266Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:49:52.476973982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:49:53.968509114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:49:53.975606011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:49:54.144206354Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-01T23:49:54.39596341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:49:55.788737535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:49:55.792969259Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:49:57.456709638Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:49:57.457751936Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:49:57.476165512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:49:57.476190955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:49:57.476200768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:49:57.476196985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:49:57.529703071Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-01T23:49:57.529729201Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=debug ts=2026-05-01T23:50:00.810711718Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-01T23:50:01.497270016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:50:01.500764935Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:50:02.476705484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:50:03.410912304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-01T23:50:03.764821359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:50:03.769416508Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:50:04.143878724Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
ts=2026-05-01T23:50:05.176991056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34168"
level=info ts=2026-05-01T23:50:05.749332524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:50:05.752979746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:50:05.912470517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:50:05.91606154Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:50:06.293187481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:50:06.296485606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:50:07.292546976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:50:07.301641733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:50:07.476237881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:50:07.477431605Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-05-01T23:50:09.248397034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:50:09.252082133Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:50:10.811428953Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=warn ts=2026-05-01T23:50:12.476920052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:50:14.144246782Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-01T23:50:14.419100868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:50:14.423284497Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:50:16.259220342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:50:16.267318102Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:50:17.47690225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:50:17.477472613Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=debug ts=2026-05-01T23:50:17.477488256Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-01T23:50:19.389476308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:50:19.398616886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:50:20.011393611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:50:20.013087808Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:50:20.810688587Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
ts=2026-05-01T23:50:20.978884582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46774"
level=warn ts=2026-05-01T23:50:22.477187013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:50:22.752724088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:50:22.755834664Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:50:22.810462995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:50:22.817718441Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:50:23.727766738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:50:23.810994454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:50:23.814107817Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:50:24.14422074Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
ts=2026-05-01T23:50:24.666122282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57302"
level=info ts=2026-05-01T23:50:27.476710294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:50:27.476720952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:50:27.476746688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:50:27.476755741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:50:27.477798154Z caller=checkpoint.go:384 msg="flushing series" totalSize="395 kB" series=2
level=info ts=2026-05-01T23:50:27.484565207Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-01T23:50:27.486617693Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009728124s
level=info ts=2026-05-01T23:50:28.684142746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:50:28.69080232Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:50:29.934101299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:50:29.937687722Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:50:32.476557696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:50:33.412605948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-01T23:50:36.648434024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:50:36.655326575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:50:37.476362272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:50:41.596003298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:50:41.601327469Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:50:42.476877663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:50:42.518054465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:50:42.5213032Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:50:45.542837855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:50:45.546132551Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-01T23:50:47.47699075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:50:47.768197701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:50:47.771661674Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-01T23:50:52.453811523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:50:52.477037536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:50:54.668451049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37952"
level=debug ts=2026-05-01T23:50:54.842361061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:50:56.072917267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:50:56.076123459Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:50:57.45735301Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:50:57.458392249Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:50:57.476751456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:50:57.476770466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:50:57.476778837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:50:57.476782478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:50:57.476778483Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:50:57.476929038Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-01T23:50:57.476961389Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:50:57.476995653Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=29.013µs success=true
level=debug ts=2026-05-01T23:50:57.477204131Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-01T23:50:59.456715539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:50:59.459937797Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:50:59.532050178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:50:59.534910664Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:50:59.860966031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:50:59.86830425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:51:00.810855781Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-01T23:51:02.476868214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:51:03.415023563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-01T23:51:03.7961407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:51:03.799328892Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:51:04.145197193Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-01T23:51:04.520746519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:51:04.524474126Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:51:04.524494603Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:51:04.524516665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:51:04.52754477Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:51:07.476727918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:07.477797005Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-01T23:51:07.505908587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:51:07.508748616Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:51:08.005197582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:51:08.012774877Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:51:10.812168383Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=debug ts=2026-05-01T23:51:10.812195977Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:51:12.476891489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:14.144811531Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-01T23:51:14.424465626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:51:14.432803721Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:51:17.03569571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:51:17.03902946Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-01T23:51:17.293972148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:51:17.29730369Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:51:17.476410944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:17.477530138Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-01T23:51:20.148139916Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:51:20.810913626Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-01T23:51:20.982264317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60330"
level=info ts=2026-05-01T23:51:21.467712901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:51:21.47152916Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:51:21.671098486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:51:21.67683084Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:51:22.476772291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:51:23.45100382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:51:23.456566826Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:51:24.144621656Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:51:27.476900226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:51:27.476935091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:51:27.476962264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:51:27.476975343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:27.47863076Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-01T23:51:27.47864486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=debug ts=2026-05-01T23:51:30.81087215Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-01T23:51:31.513219305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:51:31.517313809Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:51:32.170124673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:51:32.173135135Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:51:32.476586546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:51:33.417146409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:51:34.144000402Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-01T23:51:36.629497999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:51:36.63273565Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-01T23:51:37.476200413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:37.47733288Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-01T23:51:40.811209184Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-01T23:51:41.26020825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:51:42.476381312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:51:42.582692483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:51:42.589597259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:51:42.589618172Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:51:42.589642035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:51:42.595686653Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:51:43.716357814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:51:43.719226406Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:51:44.144459397Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-01T23:51:44.666143028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:51:44.67124664Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:51:47.476872095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:47.478033032Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-01T23:51:48.59116831Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:51:50.059274653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:51:50.066141922Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:51:50.514330312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:51:50.517335395Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:51:50.81156926Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-01T23:51:51.508458748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:51:51.511520528Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-01T23:51:52.476307029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:51:53.280563209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:51:53.284221664Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:51:54.006189685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:51:54.009844881Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:51:54.144549699Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=info ts=2026-05-01T23:51:56.762544244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:51:56.765483645Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:51:57.457200359Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:51:57.458267236Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:51:57.4766397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:51:57.476662783Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:51:57.476671612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:51:57.476667776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:51:57.477803193Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-01T23:51:59.852681461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:51:59.860143502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:52:00.810597949Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-01T23:52:02.476983119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:52:03.418277449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:52:04.144406008Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=debug ts=2026-05-01T23:52:04.144429878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-01T23:52:05.852932027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:52:05.86251026Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:52:06.699734696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:52:06.702906681Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:52:07.476899221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:07.478207468Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-01T23:52:10.811282404Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-01T23:52:12.4768271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:52:13.005855057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:52:13.011516079Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:52:14.145105938Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=info ts=2026-05-01T23:52:14.868070828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:52:14.872203858Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:52:14.872225657Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:52:14.872266815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:52:14.875377302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:52:16.497169164Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:52:17.476431257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:17.477646539Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-01T23:52:17.834386175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:52:17.84095754Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:52:18.934888925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:52:18.938254702Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:52:19.675484026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:52:19.678631198Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:52:20.810678477Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-01T23:52:20.984552009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53926"
level=warn ts=2026-05-01T23:52:22.476678725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:24.145014843Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-01T23:52:24.145037495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-01T23:52:24.674652502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51486"
level=info ts=2026-05-01T23:52:26.377056988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:52:26.384623284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:52:27.476185703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:52:27.476218924Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:52:27.476230096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:52:27.476244901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:27.477341802Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:52:29.144315621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:52:29.147242177Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:52:29.494298886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:52:29.497601707Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:52:30.436868235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:52:30.440156972Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-01T23:52:30.811362013Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-01T23:52:30.911744465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:52:30.915007747Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-01T23:52:32.476351095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:52:33.41983107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:52:34.144824836Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-01T23:52:35.536862168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:52:35.540612502Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:52:35.891569615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:52:35.89962347Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:52:36.035786204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:52:36.038672419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:52:37.440492168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:52:37.443562563Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-01T23:52:37.476526115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:37.477648304Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-01T23:52:39.717050414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:52:40.244587191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:52:40.24756681Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-01T23:52:40.81110343Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-01T23:52:41.714704612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:52:41.717741146Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-01T23:52:42.477128856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:44.144750125Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-01T23:52:46.450817585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:52:47.476800015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:47.478037816Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-01T23:52:50.811547675Z caller=checkpoint.go:372 msg="writing series" size="999 kB"
level=debug ts=2026-05-01T23:52:50.81157285Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-01T23:52:51.302701065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:52:51.310264983Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:52:52.477069496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:54.144351981Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-01T23:52:57.457720864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-01T23:52:57.457739042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:52:57.476120089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:52:57.476136811Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:52:57.476144595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:52:57.476152467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:52:57.477228692Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-01T23:53:00.811641102Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-01T23:53:00.818625346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:53:00.822054769Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-01T23:53:00.822076574Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:00.822104745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:53:00.825075357Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:53:02.47679644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:53:03.421670795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-01T23:53:03.678281163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:53:03.685149535Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:53:03.739289601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:53:03.742403894Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:53:03.742427971Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:03.74245269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:53:03.745066872Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:53:04.144341288Z caller=checkpoint.go:372 msg="writing series" size="575 kB"
level=info ts=2026-05-01T23:53:05.111957842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:53:05.11522039Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-01T23:53:05.845429146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:53:05.849089372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:53:07.476510537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:07.477569181Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-01T23:53:10.41538976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:53:10.422676107Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:53:10.810831365Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-01T23:53:12.476441244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:14.144851267Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-01T23:53:15.873859194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:53:15.877349045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-01T23:53:16.244413777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:53:17.476652401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:17.477907657Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:53:17.477923252Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-01T23:53:18.712994879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:53:18.716785277Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:53:20.810723058Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
ts=2026-05-01T23:53:20.988721621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41158"
level=info ts=2026-05-01T23:53:21.005131242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:53:21.008550315Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:53:21.368065642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:53:21.374787506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-01T23:53:22.47679587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:24.144659882Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-01T23:53:24.192698677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:53:24.195687995Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:53:24.195708445Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:24.195727923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:53:24.198029778Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:53:27.476870741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:53:27.476914245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-01T23:53:27.476913856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:53:27.476925148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:53:27.490347631Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-01T23:53:27.490367717Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-01T23:53:28.257717654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:53:28.260611054Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:53:29.02762116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:53:29.030615295Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:53:29.030642533Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:29.030667746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:53:29.033064212Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:53:30.123180378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:53:30.129818649Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-01T23:53:30.129838488Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:30.129864375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:53:30.135839463Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:53:30.382582804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:53:30.385388766Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:53:30.385409888Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:30.430577845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:53:30.433119121Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-01T23:53:30.811432117Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-01T23:53:32.4763757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:53:32.715775399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:53:32.719348689Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-01T23:53:33.42291526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:53:33.432544612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:53:34.166252955Z caller=checkpoint.go:372 msg="writing series" size="21 MB"
level=debug ts=2026-05-01T23:53:34.166279247Z caller=checkpoint.go:384 msg="flushing series" totalSize="21 MB" series=2
level=info ts=2026-05-01T23:53:34.425537276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:53:34.428798459Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:53:34.428819485Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:34.428839396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:53:34.431614265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:53:37.228590096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:53:37.231937412Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:53:37.476560342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:37.477617118Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-01T23:53:40.100152626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:53:40.103440935Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:53:40.103461868Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:40.103481554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:53:40.106103861Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:53:40.811199478Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-01T23:53:41.313612814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:53:41.324086139Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-01T23:53:42.476574056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:42.831975875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:53:43.505000118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:53:43.512841914Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:53:43.960065858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:53:43.963055361Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:53:44.144367755Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=info ts=2026-05-01T23:53:46.144798626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:53:46.147810173Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-01T23:53:47.476470784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:47.480424249Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:53:47.480455062Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=info ts=2026-05-01T23:53:49.924344943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:53:49.927917758Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:53:50.365190839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:53:50.368111293Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:53:50.368136548Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:53:50.368157769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:53:50.370617163Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-01T23:53:50.815639972Z caller=checkpoint.go:372 msg="writing series" size="6.0 MB"
level=debug ts=2026-05-01T23:53:50.815666144Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.0 MB" series=1
level=warn ts=2026-05-01T23:53:52.476813281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:53:52.533974929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:53:52.53738882Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:53:54.145192121Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-01T23:53:54.149844717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:53:54.152755881Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:53:54.583008351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:53:54.589026355Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:53:54.925932719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:53:54.929378768Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-01T23:53:57.45687472Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:53:57.457937314Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:53:57.461126214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:53:57.463749887Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:53:57.476866502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:53:57.476891327Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:53:57.476901138Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:53:57.476953669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:53:57.4799074Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-01T23:53:57.479922536Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=debug ts=2026-05-01T23:54:00.812011173Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-01T23:54:02.47629862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:54:03.424905131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-01T23:54:03.629287415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=info ts=2026-05-01T23:54:03.633520227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:54:03.637622718Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-01T23:54:03.639598134Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:54:03.639613002Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:54:03.639634633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:54:03.644819747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:54:04.144491867Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=warn ts=2026-05-01T23:54:07.47636118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:07.477444723Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:54:09.187322673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:54:09.190721715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-01T23:54:09.574981838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:54:09.582477149Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:54:09.756103593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:54:09.758841635Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-01T23:54:10.610930521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:54:10.820150518Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:54:10.820176745Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-01T23:54:12.477219562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:54:13.284218384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:54:13.290742319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:54:14.144607172Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-01T23:54:14.17027689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:54:14.173073485Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:54:14.173092926Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:54:14.173112896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:54:14.175390447Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:54:14.414936201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:54:14.417839625Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-01T23:54:17.476515783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:17.477626514Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:54:19.105274926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:54:19.111297503Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:54:20.811171551Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:54:22.4767693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:54:23.131151777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:54:23.134536409Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:54:23.893827475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:54:23.897233396Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-01T23:54:23.897256616Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:54:23.897276357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:54:23.899859039Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:54:24.065198797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:54:24.067208635Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:54:24.144712743Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-01T23:54:24.144736139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-01T23:54:25.961640385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:54:25.964799231Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:54:26.049052678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:54:26.052062199Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:54:27.476883951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:54:27.476916088Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:54:27.476925357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:54:27.476916032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:27.478543149Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=info ts=2026-05-01T23:54:30.78636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:54:30.790447927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:54:30.811526732Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-01T23:54:32.338160051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:54:32.341545623Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:54:32.476767831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:54:33.426182419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-01T23:54:34.14487918Z caller=checkpoint.go:372 msg="writing series" size="812 kB"
level=debug ts=2026-05-01T23:54:34.144901098Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-01T23:54:35.78872095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:54:35.794923781Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:54:35.794959173Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:54:35.794985689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:54:35.799342583Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:54:36.150583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-01T23:54:37.476363956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:37.477457445Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-01T23:54:38.918999567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:54:38.926108116Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:54:40.81160655Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-01T23:54:41.252898373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:54:41.256873553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-01T23:54:42.476924211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:54:43.954316887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:54:43.959118469Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-01T23:54:44.144873675Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-01T23:54:44.811468067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:54:44.817130011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:54:46.687354412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:54:46.692692669Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:54:47.476994793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:47.479413496Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:54:47.479429697Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-05-01T23:54:50.116493402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:54:50.12024186Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:54:50.811205566Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:54:52.476445948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:54:52.486611429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:54:52.494910648Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:54:52.81521482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="476 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:54:52.818574149Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:54:53.398147885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:54:53.401318725Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:54:54.144663161Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-01T23:54:56.631276573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:54:56.634591899Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:54:57.013717072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:54:57.020703121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:54:57.456811645Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:54:57.457864435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:54:57.476251902Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:54:57.476272802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:54:57.476281621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:54:57.476310573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:54:57.530107485Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-01T23:54:57.530133652Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=debug ts=2026-05-01T23:55:00.811136796Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-01T23:55:02.476908428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:55:03.428339853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:55:04.14482404Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-01T23:55:04.865737939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:55:04.869596739Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:55:05.131161538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:55:05.134752046Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:55:06.271609435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:55:06.274519Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-01T23:55:07.476402652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:55:07.477560937Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-01T23:55:08.910072714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:55:10.811860481Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-01T23:55:10.811891723Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=4
level=warn ts=2026-05-01T23:55:12.477244067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:55:13.494790505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:55:13.498273587Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:55:14.14408795Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-01T23:55:14.328579287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:55:14.333574327Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:55:17.476343885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:55:17.477813754Z caller=checkpoint.go:372 msg="writing series" size="582 kB"
level=info ts=2026-05-01T23:55:20.504824658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:55:20.511030808Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:55:20.810996212Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
ts=2026-05-01T23:55:20.994622371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57750"
level=info ts=2026-05-01T23:55:21.804628428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:55:21.807474075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-01T23:55:22.476558095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:55:24.144717624Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-01T23:55:24.191849067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:55:26.649963391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:55:26.653458806Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:55:27.476910728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:55:27.476936131Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:55:27.476967411Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:55:27.476955964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:55:27.478037171Z caller=checkpoint.go:384 msg="flushing series" totalSize="984 kB" series=4
level=info ts=2026-05-01T23:55:27.484632058Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-01T23:55:27.486243525Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009124845s
level=info ts=2026-05-01T23:55:31.915740791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:55:31.923782232Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:55:31.923808089Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:55:31.923839227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:55:31.931357499Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:55:32.47655839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:55:33.430059705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
ts=2026-05-01T23:55:35.194531153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39840"
level=warn ts=2026-05-01T23:55:37.476694868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:55:38.131168115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:55:38.713847026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:55:38.721027254Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-01T23:55:38.752173021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:55:38.754851929Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:55:39.089847497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:55:39.093315183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:55:42.154059205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:55:42.161668022Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:55:42.476428958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:55:42.692351759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:55:42.700024795Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:55:43.78799106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:55:43.791386726Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-01T23:55:44.579569637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:55:44.587059637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:55:46.33615784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:55:46.338662775Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:55:46.574828298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:55:46.578250734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:55:47.477016971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-01T23:55:52.476440162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:55:52.998842888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:55:53.001841562Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:55:57.457314375Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:55:57.458355558Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T23:55:57.476727565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:55:57.476741026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:55:57.476760344Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:55:57.476751342Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-01T23:55:57.476769152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-01T23:55:57.476898465Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-01T23:55:57.476906676Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-01T23:55:57.476925993Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.581µs success=true
level=debug ts=2026-05-01T23:55:57.477196382Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-01T23:55:58.512764731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:55:58.516685713Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:56:00.81080031Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-01T23:56:02.189394169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:56:02.192605286Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:56:02.230664713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:56:02.233163522Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:56:02.476437552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:56:03.3935728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:56:03.396388869Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-01T23:56:03.432012955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:56:03.574644277Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:56:04.14461058Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-01T23:56:06.978645297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:56:06.982783521Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-01T23:56:07.476237122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:07.477317141Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-01T23:56:10.81212882Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=debug ts=2026-05-01T23:56:10.812152578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-01T23:56:12.476304958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:14.144810833Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-05-01T23:56:15.46361474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:56:15.4693743Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:56:16.4658534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:56:16.469183556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:56:16.880001069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:56:17.476561551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:17.477682225Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-01T23:56:20.123673824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:56:20.131295357Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:56:20.811148075Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=info ts=2026-05-01T23:56:22.339136924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:56:22.342537133Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-01T23:56:22.476489695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:56:23.154498957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:56:23.160303289Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:56:24.144867113Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:56:24.462998116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:56:24.465959183Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:56:27.476932631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:56:27.476930749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:27.476972081Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:56:27.476982831Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:56:27.478934869Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-01T23:56:27.478972268Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-05-01T23:56:30.811510293Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-01T23:56:32.47669681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:56:33.279566664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:56:33.283731219Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-01T23:56:33.433563578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:56:34.144697591Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-01T23:56:35.28135433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:56:35.284916992Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:56:35.615978353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:56:35.619041548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:56:37.476331434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:37.477374607Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-01T23:56:38.530968985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:56:39.355333737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:56:39.362738196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:56:39.551470311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:56:39.558574571Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:56:40.810789502Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-01T23:56:42.47668301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:44.144175Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-01T23:56:44.210209469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:56:44.213159724Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:56:45.065673355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:56:45.069673775Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:56:47.477094665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:47.477175826Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-01T23:56:48.115610865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:56:48.118785695Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:56:50.811524072Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-01T23:56:50.999482063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40342"
level=warn ts=2026-05-01T23:56:52.476508821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:56:54.144404921Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-01T23:56:54.240341467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:56:54.243212927Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:56:57.457057305Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:56:57.458098588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-01T23:56:57.476536712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:56:57.47654538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:56:57.476563614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:56:57.476573927Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:56:57.477757016Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-01T23:56:59.717377241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-01T23:57:00.811369162Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-01T23:57:01.49012186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:57:01.492880352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:57:02.246699648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:57:02.253149146Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:57:02.476849614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:57:03.43529149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:57:04.144513919Z caller=checkpoint.go:372 msg="writing series" size="446 kB"
level=debug ts=2026-05-01T23:57:04.144537486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-01T23:57:04.29486763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:57:04.298455968Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-01T23:57:04.304500223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:57:04.307340003Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:57:04.313411734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:57:04.315811347Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-01T23:57:05.447625946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:57:05.452560991Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-01T23:57:07.477202357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:07.477427871Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=debug ts=2026-05-01T23:57:10.665364506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:57:10.810631798Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-01T23:57:12.477115582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:14.144380386Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-01T23:57:17.476660011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:17.477890404Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-01T23:57:18.258963511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:57:18.266046243Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-01T23:57:19.554477751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:57:19.561670095Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:57:20.811170641Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-01T23:57:21.851699308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:57:21.858580017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:57:22.476158697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:57:23.024367094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:57:23.027711217Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-01T23:57:24.144372023Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=debug ts=2026-05-01T23:57:24.144410699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-01T23:57:27.056681219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:57:27.060359158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:57:27.476123776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:57:27.476151036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:57:27.476160267Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:57:27.476152467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:27.477273131Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-01T23:57:30.317657022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:57:30.326405011Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:57:30.81074484Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-01T23:57:31.048550382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:57:31.051987572Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:57:32.130988275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:57:32.134325812Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-01T23:57:32.476397493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:57:32.591731981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:57:32.595167625Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:57:33.437570825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:57:34.144831444Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-01T23:57:35.200704519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52766"
level=warn ts=2026-05-01T23:57:37.476924821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:37.478046259Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-01T23:57:38.318599085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:57:38.322637296Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-01T23:57:40.810735867Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-01T23:57:40.929181571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:57:40.932618783Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-01T23:57:42.477086906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:44.024190928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-01T23:57:44.144689259Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-01T23:57:46.139535347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:57:46.143222342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:57:46.14324174Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:57:46.143262361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:57:46.146273619Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:57:46.363914275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-01T23:57:46.673982034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:57:46.676712255Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-01T23:57:47.476432459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:47.477694547Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-01T23:57:49.533005904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:57:49.538221778Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:57:50.811487553Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-01T23:57:50.811509667Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-01T23:57:52.476915119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:54.144184691Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-01T23:57:55.465809092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:57:55.472543619Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-01T23:57:55.47351255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:57:55.475953919Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:57:57.457142257Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:57:57.458171695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:57:57.476699511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:57:57.476718164Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:57:57.476729325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:57:57.476729477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:57:57.477840854Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-01T23:57:58.986480796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:57:58.990117506Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:57:59.124481817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:57:59.127415288Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:58:00.811111515Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-01T23:58:02.47724045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:58:03.439400357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:58:04.144910063Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=info ts=2026-05-01T23:58:04.653951631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:58:04.660760779Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:58:04.660780825Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:58:04.660804869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:58:04.666744459Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:58:05.040563962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:58:05.043260754Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:58:07.476319248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:07.47735119Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-01T23:58:10.810818332Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-01T23:58:12.306814887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:58:12.311098954Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:58:12.476956235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:14.144424224Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-01T23:58:16.217708061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:58:16.221311548Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:58:16.313817952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:58:16.451353618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:58:16.459456968Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-01T23:58:16.948781136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:58:16.953002533Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:58:17.476399665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:17.477671672Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-01T23:58:17.477688079Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-01T23:58:19.048994061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:58:19.058154352Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-01T23:58:20.810983407Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-01T23:58:20.907377009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:58:20.913254284Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-01T23:58:22.476362203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:58:22.478499469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:58:22.484783984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-01T23:58:23.21677482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:58:23.219921379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-01T23:58:24.144820843Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
ts=2026-05-01T23:58:24.698489504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50802"
level=info ts=2026-05-01T23:58:27.476866407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:58:27.476895262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:58:27.47690423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:58:27.476931736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:27.49253599Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-01T23:58:27.49255729Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=3
level=debug ts=2026-05-01T23:58:30.81146192Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-01T23:58:32.476896639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:58:33.441360055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:58:34.168123386Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-01T23:58:34.168147746Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
level=info ts=2026-05-01T23:58:34.872373213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:58:34.878586197Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-01T23:58:34.986739211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:58:34.992777683Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-01T23:58:36.915704313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:58:36.919250621Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-01T23:58:37.476511598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:37.477565973Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-01T23:58:38.99767918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:58:39.000594418Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-01T23:58:39.400564904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-01T23:58:39.402583016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:58:39.403672222Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:58:39.403728147Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-01T23:58:40.810875942Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-01T23:58:42.476723753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:58:43.893338256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:58:43.896533169Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-01T23:58:44.144696554Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=warn ts=2026-05-01T23:58:47.476674183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:47.480058706Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:58:47.48007347Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=debug ts=2026-05-01T23:58:48.455964754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:58:48.457532712Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T23:58:49.296140221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:58:49.40029666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:58:49.408505399Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-01T23:58:50.816681759Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-01T23:58:50.816727704Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.8 MB" series=1
ts=2026-05-01T23:58:51.004631021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37914"
level=warn ts=2026-05-01T23:58:52.477156234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:58:53.739933079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:58:53.747093919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:58:54.144479406Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
ts=2026-05-01T23:58:54.70016515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41434"
level=info ts=2026-05-01T23:58:56.240386336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:58:56.243141062Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-01T23:58:57.457514466Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:58:57.458570996Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:58:57.476965065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-01T23:58:57.476968768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:58:57.476986896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:58:57.476995344Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-01T23:58:57.479768737Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-01T23:58:57.479779168Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=2
level=info ts=2026-05-01T23:58:58.75576896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:58:58.75879265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-01T23:59:00.811585106Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-01T23:59:02.477087326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-01T23:59:03.443163147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:59:04.145222615Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-01T23:59:06.940170233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-01T23:59:06.943579176Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-01T23:59:07.476564081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:07.477656114Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-01T23:59:10.144019244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-01T23:59:10.153416576Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-01T23:59:10.815104998Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-01T23:59:10.815129911Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-01T23:59:12.476982635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:59:12.83683784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:59:12.845193354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:59:14.144693215Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-01T23:59:15.141021258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:59:15.147083093Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:59:15.147104778Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:59:15.147145801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:59:15.150022939Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-01T23:59:17.476724208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:17.477862859Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-01T23:59:18.636991319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-01T23:59:18.640077883Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-01T23:59:20.209216862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-01T23:59:20.21377783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-01T23:59:20.810512296Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-01T23:59:22.47620274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:59:22.700413192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-01T23:59:22.703351688Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-01T23:59:24.116849278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-01T23:59:24.123648586Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-01T23:59:24.144375611Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-01T23:59:24.144393923Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-01T23:59:24.534316507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-01T23:59:24.537253795Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-01T23:59:24.881389647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-01T23:59:25.922284519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-01T23:59:25.925484265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-01T23:59:27.4765258Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:59:27.476551603Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:59:27.476561186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:59:27.476584799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:27.478019223Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=info ts=2026-05-01T23:59:30.452910089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-01T23:59:30.456048321Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-01T23:59:30.810845712Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-01T23:59:31.536060087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-01T23:59:31.539319285Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-01T23:59:32.476721072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:59:32.480897039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:59:32.484074092Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-01T23:59:33.444982912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:59:34.14461184Z caller=checkpoint.go:372 msg="writing series" size="823 kB"
level=debug ts=2026-05-01T23:59:34.144634938Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-01T23:59:34.404494156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-01T23:59:34.40810203Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-01T23:59:36.249798514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-01T23:59:37.477065037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:37.47815684Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-01T23:59:40.811203153Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-01T23:59:41.130322396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:59:41.133453305Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:59:41.133472917Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:59:41.13349374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:59:41.135785658Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-01T23:59:42.477109874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-01T23:59:44.119144115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-01T23:59:44.125554681Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-01T23:59:44.144852899Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-01T23:59:44.224924013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-01T23:59:44.232109453Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-01T23:59:47.47713398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:47.479389225Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-01T23:59:47.47940498Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-05-01T23:59:50.811376894Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-01T23:59:52.476350656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:54.144825979Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-01T23:59:54.195973806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-01T23:59:54.201478016Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-01T23:59:54.545670485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:59:54.548676298Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:59:54.54869769Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-01T23:59:54.548721696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-01T23:59:54.55132902Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-01T23:59:55.714131726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-01T23:59:55.720885628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-01T23:59:57.456861408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-01T23:59:57.457914365Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-01T23:59:57.476311007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-01T23:59:57.47632612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-01T23:59:57.476333236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-01T23:59:57.476349462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-01T23:59:57.525935836Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-01T23:59:57.525975832Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-01T23:59:59.53606444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-01T23:59:59.539208321Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:00:00.265447185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:00:00.811286433Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:00:01.377735615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:00:01.38148045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-02T00:00:01.381500891Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:01.381523251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:00:01.384249556Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:00:01.624000248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:00:01.635770786Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:00:01.635804319Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:01.68207993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:00:01.693873325Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:00:02.477060541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:00:03.44585901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:00:04.144992699Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-02T00:00:05.622416433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:00:05.626291104Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:00:07.476471869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:00:07.477655481Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=debug ts=2026-05-02T00:00:10.811500835Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:00:10.811536842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-02T00:00:11.228221935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:00:11.231911143Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:00:11.231952782Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:11.231981283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:00:11.234507478Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:00:12.476542231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:12.47855307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:00:12.481888055Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:00:12.481907781Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:12.481927987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:00:12.485252514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:00:12.960625593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:00:12.964574206Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:00:12.964595132Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:12.9646157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:00:12.967616619Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:00:13.840686229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:00:13.844477028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-02T00:00:14.115161157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:00:14.118367381Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:00:14.118386557Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:14.118407592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:00:14.121216188Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:00:14.144522137Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:00:14.565803999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:00:14.571605724Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:00:15.922168907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:00:15.925657096Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:00:17.234926868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:00:17.238878305Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:00:17.476877475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:00:17.478617769Z caller=checkpoint.go:372 msg="writing series" size="461 kB"
level=info ts=2026-05-02T00:00:18.953056414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:00:18.956384851Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:00:18.956409234Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:18.956436596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:00:18.959182022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:00:19.785500236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:00:19.793374646Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:00:20.811387266Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-02T00:00:21.009537365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48530"
level=warn ts=2026-05-02T00:00:22.476975055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:22.590220655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:00:22.593507684Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:00:22.945804468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:00:22.949617505Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:00:24.143927452Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-02T00:00:24.883051199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:00:24.885848337Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:00:24.88587281Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:24.885894836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:00:24.888379516Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:00:27.331927659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-02T00:00:27.476505546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:27.476502939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:00:27.476531301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:00:27.476540615Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:00:27.47754182Z caller=checkpoint.go:384 msg="flushing series" totalSize="878 kB" series=4
level=info ts=2026-05-02T00:00:27.541654728Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-02T00:00:27.543380399Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.066304894s
level=info ts=2026-05-02T00:00:27.883874095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:00:27.887125856Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:00:28.042593115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:00:30.087586281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:00:30.091105043Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:00:32.476997337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:00:33.446886132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-02T00:00:33.906984718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:00:33.910874442Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:00:34.583640191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:00:34.587590491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:00:35.412639862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:00:35.416137229Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:00:36.421800497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:00:36.430095334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:00:37.476296077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:37.992078575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:00:37.995773359Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:00:38.391987776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:00:38.400608327Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:00:39.011405345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:00:39.015749915Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:00:42.476464262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:42.606736699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:00:42.610439527Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:00:43.501181893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:00:43.507857018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:00:43.507878434Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:00:43.507903393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:00:43.514524745Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:00:44.245455812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:00:44.249685171Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:00:47.476534304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:48.174407405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:00:48.177560749Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:00:49.574682779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:00:49.57785568Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:00:50.369499337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:00:50.377234384Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:00:52.476463714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:00:54.012671569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:00:55.218071586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:00:55.226399132Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:00:55.564915278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:00:55.567966634Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:00:55.806554701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:00:55.809342965Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:00:57.45720246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:00:57.45825455Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:00:57.45952389Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-02T00:00:57.459562544Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975199 n=1
level=debug ts=2026-05-02T00:00:57.459603781Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975199 n=1
level=info ts=2026-05-02T00:00:57.476618899Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-02T00:00:57.476622998Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:00:57.476637869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:00:57.476645604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:00:57.476677025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:00:57.476721036Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-02T00:00:57.476728524Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:00:57.476751219Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.489µs success=true
level=debug ts=2026-05-02T00:00:57.476994277Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-02T00:00:58.641823677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:00:58.645340601Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:01:00.810395782Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-02T00:01:02.476733349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:01:03.447807028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:01:04.145168391Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
ts=2026-05-02T00:01:05.212096823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36078"
level=debug ts=2026-05-02T00:01:07.184844951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-02T00:01:07.476859318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:07.478017918Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-02T00:01:08.297303853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:01:08.301131528Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-02T00:01:08.301153062Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:01:08.301186003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:01:08.304066485Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-02T00:01:08.449491547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:01:08.452881971Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:01:08.537114962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:01:08.54246817Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:01:08.542488447Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:01:08.601694893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:01:08.607132999Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:01:10.811344497Z caller=checkpoint.go:372 msg="writing series" size="567 kB"
level=debug ts=2026-05-02T00:01:10.811367578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-05-02T00:01:12.476637103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:14.144707796Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=info ts=2026-05-02T00:01:17.01286148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:01:17.018351233Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:01:17.476560987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:17.477762263Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=debug ts=2026-05-02T00:01:18.341075136Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:01:20.811372936Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-02T00:01:21.012453864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56548"
level=warn ts=2026-05-02T00:01:22.476913045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:01:23.850841713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:01:23.855168297Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:01:23.862181138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:01:23.86816123Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:01:24.143915357Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:01:24.183999377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:01:24.187591496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:01:24.615709492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:01:24.622900958Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:01:25.96873051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:01:25.972272351Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-02T00:01:27.476846113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:01:27.476847836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:27.476873283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:01:27.476882465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:01:27.47763027Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:01:27.47764118Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=info ts=2026-05-02T00:01:27.697720782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:01:27.701309831Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:01:27.950530997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:01:27.95746311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:01:30.810543754Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-05-02T00:01:32.174526779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:01:32.177976546Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:01:32.476653873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:01:32.747156846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:01:32.750888755Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-02T00:01:33.448660833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:01:34.144368609Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-02T00:01:34.915757341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:01:34.922916508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-02T00:01:35.213095866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46854"
level=warn ts=2026-05-02T00:01:37.476521863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:37.477653669Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-02T00:01:39.99016091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:01:39.993769457Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:01:40.811218237Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-02T00:01:42.476893427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:44.144093994Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:01:45.081721714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:01:45.090193439Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:01:47.476180445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:47.477269086Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-02T00:01:50.391797856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:01:50.8111262Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-02T00:01:51.014123998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59830"
level=info ts=2026-05-02T00:01:52.076880021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:01:52.08006684Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:01:52.473726309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:01:52.476756536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:54.144785583Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-02T00:01:54.381217795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:01:54.384156017Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:01:54.452577167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:01:54.455990874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-02T00:01:54.712731967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40964"
level=info ts=2026-05-02T00:01:55.733008394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:01:55.739373103Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:01:57.456650973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:01:57.45772805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:01:57.476189164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:01:57.476217411Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:01:57.476227421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:01:57.47621957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:01:57.477369913Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-02T00:02:00.810821698Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-02T00:02:01.515504057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:02:01.519240797Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:02:02.476586318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:02.906755724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:02:02.910075439Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-02T00:02:03.450124536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:02:04.144395754Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-02T00:02:04.144417682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
ts=2026-05-02T00:02:05.214545796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33566"
level=info ts=2026-05-02T00:02:06.597464925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:02:06.60059755Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:02:07.476521318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:02:07.477806575Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=info ts=2026-05-02T00:02:10.143792584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:02:10.150923117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:02:10.335396689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:02:10.33931522Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:02:10.810884264Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-02T00:02:12.476734475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:13.286377986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:02:13.289702786Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:02:14.144217523Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-02T00:02:14.754011093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:02:14.757368132Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:02:17.158558197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:02:17.161721827Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:02:17.476544562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:02:17.477714339Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-02T00:02:18.709721545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:02:18.718069158Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:02:18.718089621Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:02:18.718116145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:02:18.72507644Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:02:20.811366613Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-02T00:02:22.476653101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:23.316984553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:02:23.323651364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:02:24.14437355Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-05-02T00:02:24.144398842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-02T00:02:26.188321002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:02:27.47619127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:27.476217452Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:02:27.476239843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:02:27.47624791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:02:27.477393616Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-02T00:02:29.614491862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:02:29.619249534Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:02:30.504606017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:02:30.507839344Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:02:30.810619554Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-02T00:02:32.476855179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:02:33.452554474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:02:34.14441845Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-02T00:02:34.146525946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:02:34.149500501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-02T00:02:35.216196945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44760"
level=info ts=2026-05-02T00:02:35.909832407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=info ts=2026-05-02T00:02:35.911970067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:02:35.917539755Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:02:35.917542873Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:02:36.945388396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:02:36.953152202Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:02:37.476770677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:02:37.477835416Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-02T00:02:40.811209977Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-02T00:02:41.594169272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:02:41.598199182Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:02:42.477143166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:02:44.144185511Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-02T00:02:45.94923956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:02:45.951312205Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T00:02:47.477140414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:02:47.477324395Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-02T00:02:50.708844537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:02:50.719001854Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:02:50.81244602Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:02:50.812468866Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-02T00:02:51.814078379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:02:51.817132184Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:02:52.476883577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:53.983342679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:02:53.992407167Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:02:54.144445745Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-02T00:02:57.022815359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:02:57.457593665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:02:57.457601991Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:02:57.477041547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:02:57.477055584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:02:57.477063647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:02:57.477070286Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-02T00:02:57.477106378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:02:58.833808154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:02:58.840654168Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:03:00.252437727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:03:00.255765302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-02T00:03:00.811297212Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-02T00:03:01.453079555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:03:01.455960551Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:03:02.476414523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:03:03.45459978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:03:04.144301316Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-02T00:03:06.501999726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:03:06.505294313Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:03:07.476854231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:07.477973524Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:03:07.498063546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:03:07.501404983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:03:07.874507809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:03:07.877490899Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:03:08.193988253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:03:08.197030092Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:03:10.810573497Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=info ts=2026-05-02T00:03:12.178429792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:03:12.184421723Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:03:12.320754378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:03:12.324298848Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:03:12.477241177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:14.144141556Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-02T00:03:17.477060817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:17.477247976Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-02T00:03:17.477263328Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-02T00:03:20.810824775Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-02T00:03:22.198781107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:03:22.207286807Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:03:22.476347105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:24.144140961Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
ts=2026-05-02T00:03:24.719545487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53952"
level=info ts=2026-05-02T00:03:25.831089808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:03:25.834290086Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:03:27.476166812Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:03:27.476191816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:03:27.476200327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:03:27.476203576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:27.492321395Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-02T00:03:27.492339748Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-02T00:03:28.127378848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:03:28.132759989Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:03:30.810510266Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-02T00:03:30.86784856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:03:30.872402878Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:03:31.111934123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:03:32.476837612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:03:33.455889432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:03:34.164902501Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:03:34.164927344Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
level=info ts=2026-05-02T00:03:34.514817632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:03:34.518367456Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-02T00:03:35.219107724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57152"
level=warn ts=2026-05-02T00:03:37.476428799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:37.477546173Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-02T00:03:40.372082495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:03:40.37642288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:03:40.810876452Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-02T00:03:40.996184779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-02T00:03:42.477058029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:44.144422983Z caller=checkpoint.go:372 msg="writing series" size="500 kB"
level=info ts=2026-05-02T00:03:45.931331112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:03:45.934581085Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:03:46.154334238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:03:46.161851378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:03:47.476277881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:47.47947512Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-02T00:03:47.479490134Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=debug ts=2026-05-02T00:03:50.815146001Z caller=checkpoint.go:372 msg="writing series" size="6.0 MB"
level=debug ts=2026-05-02T00:03:50.815170667Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.0 MB" series=1
level=info ts=2026-05-02T00:03:51.274849035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:03:51.27883478Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:03:51.490527887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:03:51.494166459Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:03:51.77691695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:03:51.780275095Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:03:52.477065343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:03:53.425060359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:03:53.429732342Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-02T00:03:54.144399598Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
ts=2026-05-02T00:03:54.721593752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33660"
level=info ts=2026-05-02T00:03:55.103138559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:03:55.106322137Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:03:57.38704944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:03:57.390312059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:03:57.45664676Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:03:57.457696595Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:03:57.476106723Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:03:57.476128167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:03:57.476136437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:03:57.476136254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:03:57.479552549Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-02T00:03:57.479565245Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-02T00:03:58.35477062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:03:58.362424501Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:04:00.243771479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:04:00.246593264Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:04:00.811726124Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=debug ts=2026-05-02T00:04:00.832408398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:04:02.477029024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:04:03.456659817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-02T00:04:03.770140268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:04:03.77882926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:04:04.144298244Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=debug ts=2026-05-02T00:04:07.47711351Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=warn ts=2026-05-02T00:04:07.477135009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:04:09.859420617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:04:09.867316Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:04:10.817730926Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:04:10.81775357Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-02T00:04:12.477018079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:14.144495778Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-02T00:04:14.442426278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:04:14.446307341Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:04:17.144110193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:04:17.147517036Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:04:17.385539108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:04:17.39154999Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-02T00:04:17.476704688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:17.477864317Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:04:20.258441237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:04:20.262143706Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:04:20.810583779Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-02T00:04:21.02158821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51658"
level=warn ts=2026-05-02T00:04:22.476980384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:04:23.610483114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:04:23.615015132Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:04:23.615036886Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:04:23.615060685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:04:23.618387054Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:04:24.144152695Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-02T00:04:24.144172637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-02T00:04:24.724170794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33372"
level=info ts=2026-05-02T00:04:26.573006156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:04:26.576392469Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:04:27.476171014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:04:27.476203663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:04:27.476212872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:04:27.476215739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:27.477643355Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-05-02T00:04:29.82164133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:04:30.810596576Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:04:32.47657662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:04:33.458079956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:04:34.144348051Z caller=checkpoint.go:372 msg="writing series" size="770 kB"
level=debug ts=2026-05-02T00:04:34.144370998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-05-02T00:04:34.767726027Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:04:36.347743565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:04:36.350680797Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:04:37.476394743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:37.477431879Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:04:39.328978303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:04:39.332342707Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:04:40.810674964Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:04:41.292836162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:04:41.301891229Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:04:41.301912414Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:04:41.30195821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:04:41.309052647Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:04:41.741296149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:04:41.745411406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:04:42.476218595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:04:44.138964969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:04:44.142462246Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:04:44.144800357Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-02T00:04:47.235376497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:04:47.239997841Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:04:47.477049913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:47.479506701Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:04:47.479524956Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-05-02T00:04:48.372085264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:04:48.380498039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:04:49.618198777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:04:49.621149001Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-02T00:04:50.811296517Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-02T00:04:51.594775834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:04:51.597643166Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:04:52.477046305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:54.144351341Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-02T00:04:54.75014036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:04:54.759191954Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:04:55.166345996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:04:55.173180098Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:04:56.178537051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:04:56.187064291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:04:57.457057789Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:04:57.458098607Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:04:57.476515801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:04:57.476535596Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:04:57.476544567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:04:57.476542773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:04:57.540748771Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:04:57.540774545Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=info ts=2026-05-02T00:04:58.230614586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:04:58.242085243Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:04:58.471094082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:04:58.47844935Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:05:00.811224943Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-02T00:05:02.476877119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:05:03.459091866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:05:04.145144192Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-02T00:05:05.223355351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41762"
level=info ts=2026-05-02T00:05:05.627202544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:05:05.63312826Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:05:07.406418376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:05:07.406472098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-02T00:05:07.413903668Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:05:07.476900697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:05:07.477110896Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-05-02T00:05:10.81116063Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:05:10.811189477Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-02T00:05:12.476358338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:13.678915127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:05:13.682782344Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:05:14.144168279Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:05:15.505048412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:05:15.508349979Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:05:17.476134533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:05:17.477738687Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=info ts=2026-05-02T00:05:17.929246258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:05:17.933370357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:05:20.811413147Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-02T00:05:21.023642424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48154"
level=info ts=2026-05-02T00:05:21.887147602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:05:21.89023037Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:05:22.477137018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:22.976380039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:05:22.978932279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:05:23.6850737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:05:24.144656569Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-02T00:05:24.482020762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:05:24.505795795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-02T00:05:24.728501255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54812"
level=info ts=2026-05-02T00:05:27.476472408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:05:27.476480683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:05:27.476498763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:05:27.476507509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:05:27.477581394Z caller=checkpoint.go:384 msg="flushing series" totalSize="985 kB" series=4
level=info ts=2026-05-02T00:05:27.484018192Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-02T00:05:27.485545091Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008648545s
level=info ts=2026-05-02T00:05:29.562778839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:05:29.566545054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:05:31.024290775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:05:31.031348844Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:05:31.190646968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:05:31.193688706Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:05:32.476453471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:05:33.460300219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=warn ts=2026-05-02T00:05:37.476466192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:39.896475059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:05:39.900075891Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:05:41.044212175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:05:41.047453173Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:05:42.476676924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:42.660442324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:05:42.66419118Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-02T00:05:42.813934998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:05:42.816782896Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:05:43.269734644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="368 B" avg_comp="368 B" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" idle=1 labels="{stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:05:43.269842583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-02T00:05:43.27271655Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:05:43.630630623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:05:43.634168557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:05:43.634189928Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:05:43.634224682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:05:43.636792733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:05:45.96960705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:05:45.977234156Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:05:47.476709918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:48.74528181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:05:48.748332055Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:05:50.643261172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:05:50.646336806Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-02T00:05:51.024765432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45504"
level=info ts=2026-05-02T00:05:52.361092807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:05:52.368823012Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:05:52.477095428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:05:54.7300051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49582"
level=info ts=2026-05-02T00:05:54.754717254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:05:54.760781686Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:05:54.926141406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:05:54.928912301Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:05:57.456910877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:05:57.457965704Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:05:57.476404542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:05:57.476429304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:05:57.476439402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-02T00:05:57.476432042Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-02T00:05:57.476444328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:05:57.476602731Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-02T00:05:57.476611702Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:05:57.476628439Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.174µs success=true
level=debug ts=2026-05-02T00:05:57.476870339Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-02T00:05:59.204142762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:05:59.207516645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:06:00.811065376Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-02T00:06:01.167278111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:06:01.171034487Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:06:02.476606438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:06:02.742867542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:06:02.746135503Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
ts=2026-05-02T00:06:03.461796873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-02T00:06:03.511550878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:06:03.515321519Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:06:04.144375089Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
ts=2026-05-02T00:06:05.227223949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44584"
level=warn ts=2026-05-02T00:06:07.476922097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:07.476958739Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-02T00:06:10.098238291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:06:10.101086094Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:06:10.81115484Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-02T00:06:10.811179506Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-02T00:06:11.802254121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-02T00:06:12.476752691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:14.143915152Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=warn ts=2026-05-02T00:06:17.476988916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:17.477190186Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-02T00:06:17.614326343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:06:17.622067134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:06:17.97678075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:06:17.981652464Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:06:17.981673941Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:17.981695012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:06:17.984410128Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:06:18.077056915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:06:20.550078908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:06:20.553323188Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-02T00:06:20.553362797Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:20.553389823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:06:20.555956266Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:06:20.810851136Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
ts=2026-05-02T00:06:21.026592751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57258"
level=warn ts=2026-05-02T00:06:22.476985645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:24.144221753Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:06:26.909702629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:06:26.913039398Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:06:26.913059756Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:26.913078708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:06:26.915617046Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:06:27.282868134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:06:27.288627747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:06:27.476180108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:06:27.476206775Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:06:27.476215042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:06:27.476229622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:27.478046129Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-02T00:06:27.478061797Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-05-02T00:06:30.810800363Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=info ts=2026-05-02T00:06:30.96931477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:06:30.972623606Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:06:30.972644301Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:30.972671937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:06:30.975173944Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:06:31.339067681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:06:31.343018151Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:06:32.476666732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:06:33.136449448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:06:33.142968097Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-02T00:06:33.463199285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-02T00:06:33.819592965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:06:33.822714569Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:06:33.822735902Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:33.822775154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:06:33.825404716Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:06:34.144329272Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-02T00:06:35.474134757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:06:35.477874741Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:06:35.47789629Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:35.477917095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:06:35.480545288Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:06:35.616754119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:06:35.619841392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-02T00:06:36.430831949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:06:36.4341079Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:06:36.434129318Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:36.434154177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:06:36.436697684Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:06:37.406974993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:06:37.412194145Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-02T00:06:37.476350406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:37.477440488Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-02T00:06:37.7020813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:06:37.705031312Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:06:38.195034046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:06:38.202042324Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:06:38.202062967Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:38.202087215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:06:38.208530646Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:06:38.604715691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:06:38.608346521Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:06:38.709710556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:06:38.713434121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:06:40.813011304Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-02T00:06:42.13198918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:06:42.136875557Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:06:42.462138308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:06:42.477224677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:06:43.866271262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:06:43.86933903Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:06:44.144241028Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:06:44.187273611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:06:44.190996789Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:06:47.014872788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:06:47.018497498Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:06:47.476354904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:47.477474178Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-02T00:06:47.725419926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:06:47.728421116Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:06:49.954157887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:06:49.957061135Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:06:50.810424984Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:06:50.99920772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:06:51.00220523Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:06:51.002222964Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:06:51.002247111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:06:51.004644808Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:06:52.476154951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:06:53.549662759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:06:53.552712774Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:06:54.144540418Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-02T00:06:56.900930578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:06:56.908901913Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:06:57.457174022Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:06:57.458228328Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:06:57.476589074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:06:57.476606991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:06:57.476614734Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:06:57.476608888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:06:57.477793018Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-02T00:07:00.226362003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:07:00.229848773Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:07:00.229870137Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:07:00.229893353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:07:00.232501626Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:07:00.811159998Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-02T00:07:01.857843849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:07:01.864511814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:07:02.476725657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:07:03.464872861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:07:04.143785352Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-02T00:07:04.143808043Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=info ts=2026-05-02T00:07:04.874973198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:07:04.877791678Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:07:05.029996889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:07:05.033035721Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-02T00:07:05.230637593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39258"
level=debug ts=2026-05-02T00:07:06.934712741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:07:07.052676968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:07:07.054609158Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T00:07:07.47705438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:07.477307035Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=info ts=2026-05-02T00:07:10.599997554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:07:10.603196281Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:07:10.810980047Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-02T00:07:10.822991482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:07:10.826102355Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:07:11.018668799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:07:11.02156773Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:07:11.584124025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:07:11.587065984Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:07:12.477175539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:14.143698249Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-02T00:07:15.753822212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:07:15.76060848Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:07:16.681273259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:07:16.684399145Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-02T00:07:17.47657347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:17.477717119Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-02T00:07:17.48169409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:07:17.484573302Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:07:20.811017646Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:07:21.677843815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:07:21.680580399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:07:22.477122477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:07:23.84477463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:07:23.85147169Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:07:23.851493891Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:07:23.85151998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:07:23.856694003Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:07:24.067400267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:07:24.070154224Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:07:24.142245275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:07:24.143781572Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-02T00:07:24.143796291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=error ts=2026-05-02T00:07:24.148722378Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
ts=2026-05-02T00:07:24.736346435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36494"
level=info ts=2026-05-02T00:07:27.476387455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:07:27.476411902Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:07:27.476420316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:07:27.47643541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:27.477530319Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:07:28.506839378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:07:28.510316604Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:07:29.368498591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:07:29.37155366Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:07:30.81066111Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-02T00:07:32.476582211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:07:32.566933949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:07:32.571700484Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
ts=2026-05-02T00:07:33.466215425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:07:34.14359616Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-02T00:07:35.512064857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:07:35.517585307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:07:35.517604018Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:07:35.517624264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:07:35.522872596Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:07:35.78966362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:07:35.795960918Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:07:36.625263066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:07:36.628125043Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:07:37.476526686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:37.477625789Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-02T00:07:37.79393269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:07:37.797508219Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:07:38.330600121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:07:38.333499829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:07:39.930074488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:07:40.810279852Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-02T00:07:42.476426845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:42.8984851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:07:44.144003302Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-02T00:07:47.476846494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:47.477050795Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-02T00:07:50.07590694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:07:50.082596906Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:07:50.811312805Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:07:50.811333948Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-02T00:07:52.477047806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:07:54.045155147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:07:54.052020181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:07:54.144274951Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:07:54.307701938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:07:54.311235512Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:07:57.456746132Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:07:57.457810828Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:07:57.476186036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:07:57.476201704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:07:57.476209218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:07:57.476208084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:07:57.477312173Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-02T00:07:59.693375417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:07:59.699170107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:08:00.811283127Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-02T00:08:02.356129548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-02T00:08:02.358972087Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19de5b8bd19%3A19de5b8bd1a%3A5abb5830?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-02T00:08:02.358992399Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:08:02.359018178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:08:02.362495497Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:08:02.410581033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:08:02.41287057Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:08:02.477060378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:08:03.468087005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:08:04.143976306Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=warn ts=2026-05-02T00:08:07.476287491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:07.477399865Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:08:07.691714972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:08:07.694602231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:08:07.897707558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:08:07.900861443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:08:08.291714241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:08:08.29869782Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:08:09.538230879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:08:09.541260688Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:08:10.810872612Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-02T00:08:12.17722992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:08:12.180018615Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:08:12.476482148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:08:12.586717742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:08:12.589304616Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:08:12.752689995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:08:13.574628319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:08:13.580476291Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:08:14.144201406Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-02T00:08:14.171278771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:08:14.174341451Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:08:17.47676501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:17.477081752Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-02T00:08:17.47709691Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-02T00:08:18.192161353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:08:18.196288336Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:08:19.580307777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:08:19.583973084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:08:20.810380064Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-02T00:08:20.919702736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:08:20.926395281Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:08:21.14230149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:08:21.145333872Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:08:22.477232481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:24.143693521Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
ts=2026-05-02T00:08:24.74052184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34934"
level=info ts=2026-05-02T00:08:24.923515033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:08:24.927069428Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:08:27.476475129Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:08:27.476514115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:08:27.476532013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:08:27.476535733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:27.492208595Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-02T00:08:27.492228638Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=3
level=info ts=2026-05-02T00:08:27.84528993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:08:27.848091476Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:08:30.810652778Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-02T00:08:32.476670067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:08:33.469491768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:08:34.164280805Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:08:34.164304293Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
ts=2026-05-02T00:08:35.235174735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56092"
level=info ts=2026-05-02T00:08:37.339404396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:08:37.344652713Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:08:37.476553268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:37.477669992Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-02T00:08:40.810493986Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-02T00:08:42.471701684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-02T00:08:42.476416627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:44.144666495Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-02T00:08:46.723037695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:08:47.476847926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:47.479669538Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-02T00:08:47.47968584Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=debug ts=2026-05-02T00:08:50.814719116Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-05-02T00:08:50.814742138Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=1
ts=2026-05-02T00:08:51.033171125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37764"
level=info ts=2026-05-02T00:08:52.154545757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:08:52.158440814Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:08:52.248763223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:08:52.257445768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:08:52.476721092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:08:53.280499704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:08:53.28363296Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:08:54.144381012Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-02T00:08:54.858463768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:08:54.866645152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:08:54.866664289Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:08:54.866693092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:08:54.872810071Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:08:56.391645523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:08:56.396185209Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:08:56.771147774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:08:56.777959904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:08:57.45755841Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:08:57.457559233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:08:57.476960606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:08:57.47698005Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:08:57.476988644Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:08:57.476988073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:08:57.478295753Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-02T00:08:57.478306684Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-02T00:08:57.946239094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:08:57.951171316Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:08:58.307737582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:08:58.310696902Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:09:00.591798605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:09:00.595472557Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:09:00.811492445Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=info ts=2026-05-02T00:09:02.407979441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:09:02.411185817Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:09:02.4771697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:09:03.47053103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-02T00:09:03.787473042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:09:03.790501876Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:09:04.143793058Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-02T00:09:06.168281998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:09:06.171223613Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:09:07.476954928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:07.476965992Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-02T00:09:07.937371045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:09:07.942661153Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:09:08.637059443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:09:08.644445936Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:09:10.820300096Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:09:10.820325105Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-05-02T00:09:11.889499887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:09:11.892707851Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:09:12.476567725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:09:13.880466916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:09:13.883743941Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:09:14.145317785Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=warn ts=2026-05-02T00:09:17.476751468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:17.476845987Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-02T00:09:18.236014047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:09:18.963057659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:09:18.971037522Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:09:19.294937378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:09:20.594770611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:09:20.598082238Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:09:20.810670701Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-02T00:09:21.03474954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60922"
level=warn ts=2026-05-02T00:09:22.476400732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:09:23.217667369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:09:23.221300862Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:09:24.143862452Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-02T00:09:24.143885618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-02T00:09:27.477068317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:09:27.477064994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:09:27.47709476Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:09:27.477105888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:09:27.477742007Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=info ts=2026-05-02T00:09:28.111632945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:09:28.115906532Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:09:30.810487728Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:09:32.47680721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:09:32.80822635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:09:32.815171628Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-02T00:09:33.471839018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:09:34.145109232Z caller=checkpoint.go:372 msg="writing series" size="778 kB"
level=debug ts=2026-05-02T00:09:34.145138711Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-02T00:09:37.477059593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:37.477110789Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-02T00:09:40.810567564Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:09:41.209404283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:09:41.217890857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:09:42.067439733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:09:42.07062615Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:09:42.476674917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:44.144623798Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-02T00:09:45.942577995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:09:45.946198156Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:09:46.530071055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:09:46.53327824Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:09:47.477050654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:47.479472739Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:09:47.479498721Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-05-02T00:09:49.714392667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:09:49.721411455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:09:49.72740533Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:09:49.766600736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:09:49.77086369Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:09:50.1267392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:09:50.129929516Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:09:50.485453632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:09:50.494551892Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:09:50.810430653Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-02T00:09:50.998993618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:09:51.00445616Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:09:51.154722212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:09:51.157800578Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:09:52.106213292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:09:52.10942789Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:09:52.476426174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:54.144433653Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-02T00:09:54.74522455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39368"
level=debug ts=2026-05-02T00:09:57.456960429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:09:57.457962472Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:09:57.476370841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:09:57.47638799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:09:57.476396212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:09:57.476396171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:09:57.534746453Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:09:57.534775663Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=debug ts=2026-05-02T00:10:00.811306925Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:10:01.540256142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:10:01.550998535Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:10:02.477314361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:03.045058315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:10:03.055147395Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-02T00:10:03.473093404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:10:04.143995801Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-02T00:10:04.824885578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:10:04.835717479Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:10:07.476169279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:10:07.478620476Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-02T00:10:08.413323965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:10:08.421528307Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:10:10.811854658Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:10:10.811880189Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-02T00:10:11.538650853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:10:11.54323578Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:10:12.476464227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:10:14.143841054Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-02T00:10:16.540993224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:10:17.476412193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:10:17.478017922Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=debug ts=2026-05-02T00:10:18.108008125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:10:20.811155166Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-02T00:10:21.830602208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:10:21.834705241Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:10:22.47707144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:10:24.143598655Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
ts=2026-05-02T00:10:24.747055713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34852"
level=info ts=2026-05-02T00:10:25.136297993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:10:25.140300408Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:10:26.703928086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:10:26.711596053Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:10:27.476477568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:10:27.476500556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:10:27.476508699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:10:27.476523172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:10:27.477601457Z caller=checkpoint.go:384 msg="flushing series" totalSize="881 kB" series=4
level=info ts=2026-05-02T00:10:27.484167477Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-02T00:10:27.485719668Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008960114s
level=info ts=2026-05-02T00:10:27.492762576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:10:27.500348121Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:10:30.504272216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:10:30.507177804Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:10:31.474156941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:10:31.477463721Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:10:32.476622195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:33.183744965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:10:33.189884726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-02T00:10:33.47506703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=warn ts=2026-05-02T00:10:37.476556625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:38.226441588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:10:38.233010233Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:10:39.835387592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:10:39.838689215Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:10:40.424728542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:10:40.42787072Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:10:40.901340374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:10:40.904457342Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:10:41.450888746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:10:41.454389746Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:10:42.476989262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:45.273686848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:10:45.278835475Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:10:46.062104108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:10:47.476853979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:48.788085107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:10:48.794559751Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:10:52.476814679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:54.137177418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:10:54.140579233Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-02T00:10:54.749191645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50192"
level=info ts=2026-05-02T00:10:56.412133974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:10:56.420351757Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:10:57.456821317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:10:57.457856308Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:10:57.476257297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:10:57.476275019Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:10:57.47628354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-02T00:10:57.476283463Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-02T00:10:57.476296756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:10:57.47641929Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-02T00:10:57.476437452Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:10:57.476459699Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.274µs success=true
level=debug ts=2026-05-02T00:10:57.476674948Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-02T00:10:57.597906581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:10:57.600924522Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:10:57.785421116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:10:57.788833394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:10:59.439060757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:10:59.442311129Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:11:00.811072452Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-02T00:11:02.476705541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:11:03.476046705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:11:04.143757637Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-02T00:11:06.678301597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:11:06.682261198Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:11:06.682284541Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:11:06.682305898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:11:06.685648748Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:11:07.476880193Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-02T00:11:07.476886498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:11:09.614708551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:11:09.619987544Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:11:10.8111645Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-02T00:11:10.811187325Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-02T00:11:11.866131639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:11:11.873466124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:11:11.873485498Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:11:11.873508877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:11:11.879657515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:11:12.476988959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:12.775897233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:11:14.144394525Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=debug ts=2026-05-02T00:11:14.608482781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:11:14.610025866Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T00:11:17.476720431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:17.476891842Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=info ts=2026-05-02T00:11:18.059679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:11:18.063715465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:11:18.459828917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:11:18.462738268Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:11:18.462760229Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:11:18.462792315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:11:18.465148905Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:11:20.810282566Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=warn ts=2026-05-02T00:11:22.476651331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:11:23.32153057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:11:23.331521197Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:11:23.883657691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:11:23.889243972Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-02T00:11:24.144012711Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-02T00:11:24.751022369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49498"
level=info ts=2026-05-02T00:11:25.885242104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:11:25.891984049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:11:26.11955723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:11:26.12325585Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:11:26.327974388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:11:26.331288336Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:11:27.477027683Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:11:27.477051889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:11:27.477059612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:11:27.477097191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:27.477989099Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-02T00:11:27.478002263Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=info ts=2026-05-02T00:11:27.626567283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:11:27.629787939Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:11:30.810896211Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-02T00:11:32.477056881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:11:32.692053737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:11:32.695588058Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-02T00:11:33.477790386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-02T00:11:33.490827802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:11:33.496604692Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:11:34.144142865Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=warn ts=2026-05-02T00:11:37.476763817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:37.476832581Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-02T00:11:37.77164933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:11:37.775613129Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:11:37.781654031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:11:37.788524188Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:11:37.835668759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:11:37.83841152Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:11:39.146812757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:11:39.14989686Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:11:40.810698805Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-02T00:11:41.69629604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:11:41.699346987Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:11:42.477060252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:43.514255991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:11:44.144094984Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:11:45.445317229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:11:45.449550919Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:11:47.476767658Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-02T00:11:47.476781155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:11:47.668539705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:11:47.67211901Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:11:48.528591067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:11:48.532099476Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:11:49.324295709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:11:49.327147701Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:11:49.517960918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:11:49.521469566Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:11:50.811014884Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-02T00:11:52.476701946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:11:53.318702379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:11:53.321568274Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:11:54.143996971Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-02T00:11:54.242401311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:11:57.456587749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:11:57.457641327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:11:57.477042431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:11:57.477057894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:11:57.47706542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:11:57.47709065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:11:57.477110937Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-02T00:12:00.810736691Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-02T00:12:02.47634103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:12:03.479264658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:12:04.143863228Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=debug ts=2026-05-02T00:12:04.143885731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-02T00:12:04.381408702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:12:04.390383259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-02T00:12:05.244771678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52410"
level=info ts=2026-05-02T00:12:06.461250243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:12:06.467734929Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-02T00:12:07.47620879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:07.477580906Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-05-02T00:12:10.810459566Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-02T00:12:12.280391525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:12:12.288508231Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:12:12.477078033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:12:12.677350322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:12:12.680731926Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:12:12.991930003Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:12:13.538193385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:12:13.541804501Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:12:14.143463913Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-02T00:12:17.476508922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:17.477780518Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-02T00:12:18.914960415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:12:18.920118024Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:12:20.810526751Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-02T00:12:21.04378212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41284"
level=info ts=2026-05-02T00:12:21.275649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:12:21.279469208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:12:21.396608325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:12:21.39967716Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:12:22.152836197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:12:22.160892727Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:12:22.476892432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:12:23.509688841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:12:23.517189766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:12:23.860287681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:12:23.86361017Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:12:24.143554869Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-02T00:12:24.14357723Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-02T00:12:24.75536358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42428"
level=info ts=2026-05-02T00:12:27.476344512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:12:27.47636968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:12:27.476377879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:12:27.476390209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:27.477483598Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:12:28.107532829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:12:28.110505775Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:12:28.339933756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:12:28.353350058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:12:28.356823549Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:12:30.478014256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:12:30.481416237Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:12:30.81016029Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-02T00:12:32.217867039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:12:32.22210472Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:12:32.476978431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:12:33.481093591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:12:34.143495508Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-02T00:12:34.217637612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:12:34.220840804Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:12:35.623610904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:12:35.627089925Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:12:35.627110125Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:12:35.627130078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:12:35.629726888Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:12:35.920683698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:12:35.924298194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:12:35.924320036Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:12:35.927504481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:12:35.930546613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:12:37.477102347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:37.477112151Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-02T00:12:40.810764726Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-02T00:12:42.223564606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:12:42.227892361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:12:42.476618316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:44.143813615Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-02T00:12:47.476621569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:12:47.476892269Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-02T00:12:48.356588823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:12:49.090668631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:12:49.095995968Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-02T00:12:49.881136013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:12:49.884332646Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:12:50.811590738Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:12:50.811611546Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
ts=2026-05-02T00:12:51.045311184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48434"
level=warn ts=2026-05-02T00:12:52.476190281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:12:53.827975722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:12:53.831683553Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:12:54.14376821Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:12:54.542324395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:12:54.549628193Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:12:57.457568396Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:12:57.457573119Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:12:57.476978887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:12:57.476997842Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:12:57.477006522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:12:57.477027172Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-02T00:12:57.477027206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:00.810587756Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-02T00:13:02.061541359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:13:02.065330788Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:13:02.065356302Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:02.06538387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:13:02.067925285Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:13:02.476731942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:13:03.48291126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:13:04.143599498Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-02T00:13:05.636679418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:13:05.643586278Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:13:05.643610804Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:05.643642507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:13:05.650488187Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:13:06.031378311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:13:06.03424104Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:13:07.270126741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:13:07.276070841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:13:07.47655885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:07.47664474Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:13:08.961503841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:13:08.964502224Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-02T00:13:08.964523522Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:08.964546016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:13:08.96682308Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:13:09.123301822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:13:09.126155488Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:13:09.126173884Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:09.264690493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:13:09.267506176Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:13:09.894272492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:13:09.901199999Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:13:10.810254133Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-02T00:13:11.592456252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:13:11.595421031Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:13:11.595441035Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:11.595463776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:13:11.597774223Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-02T00:13:12.477133938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:14.144135367Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-02T00:13:14.210209212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:13:14.213421672Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:13:14.213447634Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:14.21347405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:13:14.215744626Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:13:15.727983949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:13:15.73111384Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:13:15.731136924Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:15.731162455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:13:15.733596593Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:13:16.786123215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:13:16.788820125Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:13:17.476534424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:17.477839134Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-02T00:13:18.407409902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:13:18.410386067Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:13:18.670305717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:13:18.672903099Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:13:18.672922878Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:18.672968211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:13:18.67507739Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:13:19.429381498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:13:20.039157289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:13:20.04196085Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:13:20.81049778Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-05-02T00:13:20.989058115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:13:20.991826188Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-02T00:13:21.047007926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45530"
level=info ts=2026-05-02T00:13:22.360482123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:13:22.363703293Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:13:22.476523345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:13:22.510644699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:13:22.513612034Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:13:23.679599782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:13:24.143868584Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-02T00:13:24.143892926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-02T00:13:25.277139707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:13:25.284084879Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:13:26.108362276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:13:26.111342176Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:13:26.111366355Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:26.111390498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:13:26.113987151Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:13:27.476372417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:13:27.476401055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:13:27.476410633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:13:27.476423783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:27.496038969Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-02T00:13:27.496065387Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=1
level=info ts=2026-05-02T00:13:27.960084238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:13:27.963824314Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:13:28.97269777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:13:28.975430267Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:13:28.975445916Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:28.975465776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:13:28.977836571Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:13:29.074149541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:13:29.076566292Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:13:30.810475828Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-02T00:13:32.476982886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:13:33.483786445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-02T00:13:33.684964448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:13:33.690403866Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:13:34.164167111Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:13:34.164192145Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
level=info ts=2026-05-02T00:13:35.202069636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:13:35.205053574Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-02T00:13:35.24921587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58822"
level=info ts=2026-05-02T00:13:35.620732177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:13:35.628108421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:13:37.476490249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:37.477578792Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-02T00:13:37.655971347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:13:37.658686535Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:13:40.810270337Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-02T00:13:42.476651392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:44.144670193Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-02T00:13:44.179260125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:13:44.182911832Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:13:45.412805582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:13:45.418403927Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19de5b8bd0d%3A19de5eb0519%3Ac4674547?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-02T00:13:45.418425602Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:13:45.418448245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:13:45.421657031Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-02T00:13:45.433732461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:13:45.436282811Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:13:47.477041418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:47.479222114Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-02T00:13:47.479238869Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=4
level=info ts=2026-05-02T00:13:49.251618634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:13:49.254737744Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:13:49.819138118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:13:49.824044481Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:13:50.816981023Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-05-02T00:13:50.817007327Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=1
level=warn ts=2026-05-02T00:13:52.476832821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:13:53.514702621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:13:53.517839233Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:13:53.677444633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:13:54.144125768Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-02T00:13:54.981230421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:13:54.985590625Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:13:55.297451754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:13:55.305291598Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:13:56.531238487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:13:56.534785596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:13:57.457112647Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:13:57.458166391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:13:57.476498914Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:13:57.476515424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:13:57.476524165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:13:57.476523403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:13:57.480024766Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-02T00:13:57.480036674Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-02T00:13:57.924964203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:13:57.928786669Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-02T00:13:58.183528426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:13:58.186742794Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:14:00.810556067Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-02T00:14:02.47645401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:14:03.025796156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:14:03.032554364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:14:03.032576395Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:14:03.03260282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:14:03.03957511Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-02T00:14:03.485213863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:14:04.144576044Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-02T00:14:05.223400379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:14:05.227073023Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:14:06.595299197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:14:06.605127968Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:14:07.476873847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:07.476892033Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-02T00:14:08.971676518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:14:08.974714851Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:14:09.077130721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:14:09.080213239Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:14:10.374486665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:14:10.378525143Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:14:10.816176857Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:14:10.816201442Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-02T00:14:12.477112069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:14.144488423Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-02T00:14:14.147968261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:14:14.154627049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:14:15.118406985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:14:15.121172379Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:14:17.476837657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:17.476869275Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:14:17.533965896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:14:17.537089854Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:14:20.652896733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:14:20.81021355Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=info ts=2026-05-02T00:14:21.389256568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:14:21.392269653Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:14:22.4763236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:22.816363729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:14:24.144490744Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-02T00:14:24.144515637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-02T00:14:27.476271344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:14:27.476274165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:27.476301352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:14:27.476312476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:14:27.477955363Z caller=checkpoint.go:372 msg="writing series" size="639 kB"
level=info ts=2026-05-02T00:14:29.700889915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:14:29.707574782Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:14:30.756057544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:14:30.760680101Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:14:30.810764193Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:14:32.477074333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:14:33.451452378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:14:33.455308911Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-02T00:14:33.486641029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-02T00:14:33.642541687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:14:33.646603479Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:14:34.144553569Z caller=checkpoint.go:372 msg="writing series" size="797 kB"
level=debug ts=2026-05-02T00:14:34.144576243Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-02T00:14:37.476383185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:37.477500093Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:14:37.704989014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:14:37.709340338Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:14:37.919749523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:14:37.923151382Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:14:40.331179885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="30 MB" avg_uncomp="15 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:14:40.337025332Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:14:40.810413609Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:14:42.029539544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:14:42.036899012Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:14:42.47649817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:14:43.830195655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:14:43.83324712Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:14:44.144500158Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-02T00:14:47.476501212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:47.479201851Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:14:47.47921676Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-05-02T00:14:50.810394639Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-02T00:14:51.919712128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:14:51.927229036Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:14:52.476342808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:14:53.483627947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:14:53.486834162Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:14:53.602171749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:14:54.143800152Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-02T00:14:54.766532219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51608"
level=info ts=2026-05-02T00:14:56.092889129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:14:56.098351465Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:14:56.098372312Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:14:56.098397579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:14:56.10206879Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:14:57.334472798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:14:57.338252751Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:14:57.457364139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:14:57.458408384Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:14:57.476757508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:14:57.47677736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:14:57.476786248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:14:57.476788547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:14:57.545110662Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:14:57.54517251Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=info ts=2026-05-02T00:14:58.239359956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:14:58.253738679Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:14:59.774751901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:14:59.792188655Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:15:00.811689368Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:15:02.07179063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:15:02.087670793Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:15:02.476429616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:15:03.487751439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:15:04.144044731Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-02T00:15:05.234198784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:15:05.242321621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:15:07.476201176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:15:07.476828473Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-02T00:15:09.320423095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:15:09.322020055Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-02T00:15:10.241330696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:15:10.244786176Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:15:10.811073672Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:15:10.811094976Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-02T00:15:12.476493764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:15:14.143584034Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:15:14.964984345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:15:14.969622504Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:15:15.279483985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:15:15.287296361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:15:17.382698065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:15:17.385609641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:15:17.476380906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:15:17.478083393Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=info ts=2026-05-02T00:15:17.910935982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:15:17.914561055Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-02T00:15:19.2512743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:15:19.254036288Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:15:20.085198524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:15:20.088358897Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:15:20.8105748Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
ts=2026-05-02T00:15:21.052969866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49662"
level=warn ts=2026-05-02T00:15:22.477004204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:15:22.999128163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:15:24.143745655Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-02T00:15:24.80952073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:15:24.81469804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:15:25.22383508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:15:25.231734823Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:15:27.29254738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:15:27.30018574Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:15:27.476677935Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=4
level=info ts=2026-05-02T00:15:27.476701849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:15:27.476710288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:15:27.476725836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:15:27.47673484Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-02T00:15:27.481258968Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-02T00:15:27.483340618Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006772088s
level=warn ts=2026-05-02T00:15:32.477118166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:15:33.489711259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:15:35.256204451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44334"
level=info ts=2026-05-02T00:15:37.384148787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:15:37.388794061Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:15:37.476921841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:15:39.268543688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:15:39.271623829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:15:40.01148596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:15:40.014543028Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:15:41.174747724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:15:41.179410709Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:15:42.477112793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:15:42.592443344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:15:42.595590066Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:15:45.011733521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:15:45.014714631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:15:47.476496238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:15:48.766397284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:15:48.769453586Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:15:49.133443298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:15:49.140963322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:15:49.140983449Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:15:49.141011818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:15:49.147913016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:15:49.754822928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:15:49.757983054Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:15:50.295444102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:15:50.29911401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:15:52.195191567Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:15:52.476613218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:15:54.770296742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58016"
level=info ts=2026-05-02T00:15:55.660831573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:15:55.665838587Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:15:57.456690904Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:15:57.457742167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:15:57.46012257Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-02T00:15:57.460179641Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975200 n=1
level=debug ts=2026-05-02T00:15:57.460234514Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975200 n=1
level=info ts=2026-05-02T00:15:57.476184898Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-02T00:15:57.476191194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:15:57.476217951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:15:57.476235666Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:15:57.476249169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-02T00:15:57.47630754Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-02T00:15:57.476319044Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:15:57.476337818Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.961µs success=true
level=debug ts=2026-05-02T00:15:57.476582291Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:16:00.518870484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:16:00.526358579Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:16:00.810179911Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-02T00:16:01.518756339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:16:02.456002298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:16:02.464337696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:16:02.476501272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:16:03.491214212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:16:04.143899753Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=warn ts=2026-05-02T00:16:07.476757706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:07.47680705Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-02T00:16:09.248305316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:16:09.252802056Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:16:10.811259073Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-02T00:16:10.811287137Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-02T00:16:11.561313538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:16:11.565176769Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:16:12.476738718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:14.143879079Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-02T00:16:14.523854767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:16:14.527086508Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:16:16.438419596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:16:17.330132675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:16:17.333210645Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:16:17.477182119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:17.477419507Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-02T00:16:20.810970178Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=warn ts=2026-05-02T00:16:22.476764523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:16:23.6665318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:16:23.670497388Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:16:24.143778654Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:16:24.41544027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:16:24.422765947Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:16:24.941249089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:16:24.948442854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:16:26.820513835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:16:26.827036466Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:16:27.476491232Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:16:27.476502475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:27.476529729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:16:27.476546702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:16:27.477358981Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-02T00:16:27.477373312Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=5
level=info ts=2026-05-02T00:16:30.723520016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:16:30.737154116Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:16:30.810468334Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-02T00:16:31.442143808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:16:31.445507572Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:16:32.476370972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:16:32.68239939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:16:32.685488622Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-02T00:16:33.493009649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:16:34.143541539Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-02T00:16:34.724234046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:16:34.727227623Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:16:37.476576998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:37.476629607Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-02T00:16:37.71838668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:16:37.721597361Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:16:38.310364887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:16:38.311878257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:16:38.31323784Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:16:38.315382044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:16:40.810893877Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-02T00:16:41.840352274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:16:42.477074369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:16:42.508258963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:16:42.512130835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:16:43.225359242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:16:43.228668403Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:16:44.143799539Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:16:45.389839933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:16:45.392907504Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:16:47.477071532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:47.477103576Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-02T00:16:48.489622499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:16:50.810489516Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:16:52.463140615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:16:52.46969013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:16:52.476829199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:54.143959672Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-02T00:16:57.457262067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:16:57.458315357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:16:57.476663858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:16:57.476679522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:16:57.476690218Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:16:57.476698655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:16:57.476760623Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-02T00:17:00.810388485Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-02T00:17:02.082959171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:17:02.087640734Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:17:02.476358526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:17:03.494157713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:17:04.14390632Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=debug ts=2026-05-02T00:17:04.143930495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=info ts=2026-05-02T00:17:05.530769109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:17:05.534624325Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:17:07.476554404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:07.476819931Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-02T00:17:08.52662782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:17:08.530498641Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:17:09.911491215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:17:10.26849192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:17:10.27244843Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:17:10.810972375Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-02T00:17:12.436572627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:17:12.442551633Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:17:12.476707456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:17:12.485814948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:17:12.488678633Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:17:13.050833703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:17:13.05760686Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:17:14.143479413Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-02T00:17:17.476139763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:17.477300272Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-02T00:17:19.159772827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:17:19.163986429Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:17:19.647765831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:17:19.650952677Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:17:20.810148296Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:17:22.061546079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:17:22.068148426Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:17:22.476956848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:24.144218019Z caller=checkpoint.go:372 msg="writing series" size="319 kB"
level=debug ts=2026-05-02T00:17:24.144239202Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-02T00:17:24.776520033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49456"
level=info ts=2026-05-02T00:17:25.209157214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:17:25.212977823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:17:26.393928851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:17:26.397265686Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:17:26.46059319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:17:26.463224029Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:17:27.476148441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:17:27.476177112Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:17:27.476185814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:17:27.476244647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:27.477370623Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:17:28.548745988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:17:28.551845409Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:17:29.472289458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:17:29.475026881Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:17:30.511101409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:17:30.810783125Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-02T00:17:31.931991062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:17:31.93552484Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:17:32.476769136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:17:33.495027565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:17:34.143618837Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-02T00:17:34.145729035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:17:34.150578244Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:17:34.242705945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:17:34.245465121Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:17:37.47643763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:37.476545512Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-02T00:17:40.810526426Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-02T00:17:40.877837034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:17:40.884814218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:17:40.884838754Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:17:40.884866705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:17:40.891337156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:17:41.666312673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:17:42.47675587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:44.143998671Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-02T00:17:47.476857863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:47.477116834Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-02T00:17:50.811380625Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:17:50.811403914Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-02T00:17:52.477010126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:54.143638628Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:17:55.100362767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:17:55.105831278Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:17:56.947076332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:17:56.954233388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:17:57.266230854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:17:57.272274247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:17:57.456780183Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:17:57.457845662Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:17:57.476287004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:17:57.476305179Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:17:57.476312656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:17:57.476302633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:17:57.477430275Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-02T00:17:58.930174783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:17:58.933396364Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:17:58.93343312Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:17:58.933462507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:17:58.936153774Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:18:00.262523297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:18:00.265684648Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:18:00.81012172Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-02T00:18:02.151916499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:18:02.155717426Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:18:02.476718399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:18:03.30953176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:18:03.312663392Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-02T00:18:03.496601014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:18:04.144479105Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-02T00:18:04.665798617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:18:04.670661239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:18:04.670681233Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:18:04.670704896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:18:04.673647331Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:18:06.85717735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:18:07.281213365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:18:07.285230793Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:18:07.476821853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:07.476870698Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:18:08.332687834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:18:08.339105214Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:18:10.810899965Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-02T00:18:11.047659928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:18:11.054419042Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:18:12.476731777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:18:13.10273267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:18:13.10614432Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:18:14.144221108Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-02T00:18:14.389767372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:18:14.392870776Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:18:17.292275114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:18:17.295036635Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:18:17.346462155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:18:17.349104082Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:18:17.47707328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:17.477249258Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-02T00:18:17.498214205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:18:17.5015004Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:18:18.532831929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:18:18.536038309Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:18:19.493889728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:18:19.496856138Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:18:20.810551368Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-05-02T00:18:20.810576043Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-05-02T00:18:21.064376161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44898"
level=warn ts=2026-05-02T00:18:22.476962359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:18:23.26037009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:18:23.263597046Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:18:23.961485405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:18:23.964520909Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:18:24.144209559Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
ts=2026-05-02T00:18:24.780537684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38080"
level=info ts=2026-05-02T00:18:25.619707832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:18:25.623002329Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:18:27.476331393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:18:27.476366303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-02T00:18:27.47635587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:18:27.476374417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:18:27.498971989Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-02T00:18:27.49899218Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=2
level=debug ts=2026-05-02T00:18:28.701278532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:18:30.811060813Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-02T00:18:31.65171527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:18:31.659100359Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:18:32.476851917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:18:33.49815764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:18:34.165350523Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:18:34.165374931Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
ts=2026-05-02T00:18:35.26461298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60448"
level=debug ts=2026-05-02T00:18:37.256174698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:18:37.476518848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:37.47659948Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-02T00:18:39.663296298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:18:39.669173185Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:18:40.810697128Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-02T00:18:42.062092063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:18:42.065268231Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:18:42.476920182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:44.144367656Z caller=checkpoint.go:372 msg="writing series" size="681 kB"
level=info ts=2026-05-02T00:18:44.686457929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:18:44.689715458Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:18:47.476365406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:47.479473926Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-02T00:18:47.479491206Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=4
level=info ts=2026-05-02T00:18:49.853244846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:18:49.857169747Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:18:50.816614414Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-05-02T00:18:50.816646249Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=1
level=info ts=2026-05-02T00:18:50.957432031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:18:50.965047074Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-02T00:18:51.066175582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39392"
level=info ts=2026-05-02T00:18:52.052135172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:18:52.05633988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:18:52.476894912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:18:53.172240098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:18:53.182535195Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:18:53.567217211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:18:53.575426814Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:18:53.967566226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:18:53.97055267Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:18:54.144516548Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
ts=2026-05-02T00:18:54.782122577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57594"
level=info ts=2026-05-02T00:18:57.457696517Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:18:57.457726418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:18:57.476098429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:18:57.47611319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:18:57.476120839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:18:57.476164588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:18:57.479968597Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-02T00:18:57.479981609Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-02T00:18:58.390993689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:18:58.394317314Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:19:00.066336617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:19:00.07082948Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:19:00.81124353Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=warn ts=2026-05-02T00:19:02.477057532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:19:03.500078712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:19:03.675415532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:19:03.675475476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:19:03.679097373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:19:04.143544182Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-02T00:19:06.017616574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:19:06.020851783Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:19:06.322715327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:19:06.325850559Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:19:06.325871306Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:06.325893451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:19:06.328469392Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:19:07.476214959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:07.477341616Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-02T00:19:08.78623932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:19:08.789717824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:19:08.789738667Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:08.789776743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:19:08.792457587Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:19:08.967459297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:19:08.969231308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-02T00:19:09.320290526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:19:09.322913095Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:19:10.81615606Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:19:10.816178742Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-05-02T00:19:11.101445323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:19:11.105323379Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:19:12.476282959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:19:12.981378234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:19:12.984095803Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:19:13.340000291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:19:13.347598506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:19:14.144849861Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-02T00:19:16.415286752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:19:16.418299428Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-02T00:19:17.476255068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:17.477322538Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:19:18.131154514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:19:18.134123695Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:19:18.849294757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:19:18.852128036Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:19:20.810378184Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-02T00:19:21.067456509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59918"
level=warn ts=2026-05-02T00:19:22.476497559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:19:22.830285361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:19:22.833350293Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:19:24.143639239Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-02T00:19:24.14366085Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-02T00:19:25.129145026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:19:25.131971474Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:19:25.1319936Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:25.132015437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:19:25.134197464Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:19:26.660359579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:19:26.663911467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:19:27.476668581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:19:27.476703532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:19:27.476714728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:19:27.476707768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:27.477153823Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-02T00:19:30.810859009Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:19:32.477200257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:19:33.420261352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:19:33.424087218Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-02T00:19:33.502171436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:19:34.14418747Z caller=checkpoint.go:372 msg="writing series" size="810 kB"
level=debug ts=2026-05-02T00:19:34.144213446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-02T00:19:36.008370133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:19:37.436781232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:19:37.440335693Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:19:37.4764216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:37.477523261Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:19:38.068073619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:19:38.071362281Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:19:38.204615397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:19:38.211352352Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:19:40.810120312Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:19:41.426253524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:19:41.431423244Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-02T00:19:41.431443535Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:41.431465181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:19:41.433809091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:19:42.477193659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:19:43.380565224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:19:43.387495771Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:19:43.387516401Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:43.387542295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:19:43.39439364Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:19:44.143677521Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-02T00:19:44.337926929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:19:44.340925328Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:19:44.875139153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:19:44.878012068Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:19:45.924074948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:19:45.931536591Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:19:46.809841491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:19:46.813371458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:19:47.476186322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:47.479381865Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:19:47.479397372Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-05-02T00:19:50.810882099Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-02T00:19:52.215188521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:19:52.218305982Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:19:52.218327202Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:52.218347749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:19:52.22066657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:19:52.476189197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:54.143670507Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-02T00:19:57.090552834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:19:57.099779221Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:19:57.246261241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:19:57.250189195Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:19:57.250210237Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:19:57.250231191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:19:57.252578338Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:19:57.456654241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:19:57.457699244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:19:57.47703115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:19:57.477047382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:19:57.477055117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:19:57.477061861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:19:57.542995596Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:19:57.543061201Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=info ts=2026-05-02T00:19:58.704695806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:19:58.716162267Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-02T00:19:58.746879772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:19:58.761198396Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:20:00.81100044Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:20:01.049417549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:20:01.058992146Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:20:01.308339745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:20:01.308410485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" idle=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:20:01.354625435Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:20:01.662583336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:20:01.670530841Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:20:01.670586507Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:01.670634478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:20:01.683880836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:20:02.243980262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:20:02.255562167Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:20:02.47711062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:20:02.963399982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:20:03.505299321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:20:04.144818263Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-02T00:20:04.326623393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:20:04.340654487Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:20:07.137286828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:20:07.1461179Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:20:07.146152463Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:07.146183783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:20:07.161508634Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:20:07.476226552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:20:07.476766455Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-02T00:20:09.024496229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:20:09.027654203Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:20:09.02768112Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:09.027705675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:20:09.03068682Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:20:10.195083884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:20:10.198326744Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:20:10.198350008Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:10.198371722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:20:10.200869437Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:20:10.810758071Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:20:10.810790004Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-02T00:20:12.477175213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:13.895175239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:20:13.900186738Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:20:14.143856916Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:20:16.742063801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:20:16.745670972Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:20:17.476955378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:20:17.47751246Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=info ts=2026-05-02T00:20:18.162763878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:20:18.16588091Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:20:18.494801295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:20:18.501600317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:20:18.501622216Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:18.501647472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:20:18.507848862Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:20:19.023165279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:20:19.02751634Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:20:19.291355633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:20:19.294366319Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:20:19.978318998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:20:19.98147674Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:20:20.810930598Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
ts=2026-05-02T00:20:21.069739686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50784"
level=warn ts=2026-05-02T00:20:22.476977617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:23.22407458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:20:23.2276551Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:20:23.559849819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:20:23.563276864Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:20:24.144097812Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-02T00:20:26.298492986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:20:26.301633817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:20:27.47610114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:20:27.476137976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:20:27.476146127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:20:27.476148317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:20:27.477202657Z caller=checkpoint.go:384 msg="flushing series" totalSize="672 kB" series=4
level=info ts=2026-05-02T00:20:27.483489234Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-02T00:20:27.48513142Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008666781s
level=info ts=2026-05-02T00:20:27.888986282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:20:27.891800718Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:20:27.891818998Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:27.89183839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:20:27.894067538Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:20:28.700712419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:20:28.707955916Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:20:29.061949662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:20:29.064712087Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:20:30.595546759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:20:30.60229922Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:20:31.965521811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:20:31.970090003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:20:32.477175039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:33.455843531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:20:33.458924782Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-02T00:20:33.509403529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-02T00:20:34.317168507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:20:34.320504258Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:20:36.877149544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:20:36.88478393Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:20:37.122546373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:20:37.122596718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:20:37.126730608Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:20:37.423666156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="917 kB" avg_uncomp="917 kB" idle=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:20:37.427256141Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=warn ts=2026-05-02T00:20:37.476386722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:39.462223079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:20:39.466410943Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:20:39.531475702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:20:39.534235316Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:20:42.476987549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:45.46926871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:20:45.476469829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:20:47.34976122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:20:47.352791724Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:20:47.47701914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:20:49.720792542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:20:49.91629215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:20:49.919345126Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:20:51.490470589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:20:51.494413014Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-02T00:20:52.476720599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:52.656600185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:20:52.660194297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:20:52.660214852Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:20:52.660238826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:20:52.663315854Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-02T00:20:54.598991275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:20:54.603098413Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
ts=2026-05-02T00:20:54.789425547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49584"
level=info ts=2026-05-02T00:20:55.169170851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:20:55.172316495Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:20:55.581423475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:20:55.584526312Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:20:56.238059516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:20:56.241239086Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:20:57.457167617Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:20:57.458210597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:20:57.476560758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:20:57.476583968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:20:57.47659285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-02T00:20:57.476599573Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-02T00:20:57.476621105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:20:57.476741913Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-02T00:20:57.476754041Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:20:57.476787581Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.118µs success=true
level=debug ts=2026-05-02T00:20:57.477062306Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:20:59.659147394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:20:59.662551096Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:20:59.843047591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:20:59.850766768Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:21:00.810821881Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-02T00:21:01.572789828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:21:01.575988754Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:21:02.476353369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:21:03.510852246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:21:04.144322402Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=debug ts=2026-05-02T00:21:04.324443809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:21:05.923955827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:21:05.928958035Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:21:06.152697162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:21:06.155547632Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-02T00:21:07.476588577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:07.477656357Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-02T00:21:08.999160941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:21:09.001931995Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:21:09.067280893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:21:09.069725612Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:21:09.769743952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:21:09.773276392Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:21:10.811737816Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-02T00:21:10.811760977Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-02T00:21:12.477122657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:14.144274474Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=info ts=2026-05-02T00:21:14.349689088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:21:14.354351246Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-02T00:21:17.477136799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:17.477425263Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=info ts=2026-05-02T00:21:17.819310521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:21:17.823815814Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:21:18.813338003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:21:18.816618739Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:21:20.811456795Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=info ts=2026-05-02T00:21:22.332403256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:21:22.336905693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:21:22.336927532Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:21:22.336979964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:21:22.337520685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:21:22.340675416Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:21:22.340676816Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:21:22.477247506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:24.143994457Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-02T00:21:24.791792498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60786"
level=info ts=2026-05-02T00:21:27.088694818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:21:27.09652616Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:21:27.476374458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:21:27.476384709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:21:27.476408527Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:21:27.476417053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:21:27.478379081Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-02T00:21:27.478391907Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=info ts=2026-05-02T00:21:28.220068458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:21:28.223839938Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:21:30.201522111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:21:30.20496415Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:21:30.811303352Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-02T00:21:31.804115993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:21:31.808060715Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:21:32.476525248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:21:33.512523921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:21:34.143973952Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-02T00:21:34.715232694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:21:34.722211316Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:21:34.855683278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:21:34.863774794Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
ts=2026-05-02T00:21:35.27292485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37512"
level=info ts=2026-05-02T00:21:35.624552577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:21:35.629081141Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:21:37.476590425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:37.477658601Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-02T00:21:37.65806022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:21:37.661955419Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:21:38.88505684Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:21:40.810863824Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-02T00:21:41.074342729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:21:41.71418299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:21:41.717932026Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:21:42.477206755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:44.144215222Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:21:44.428768925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:21:44.431800062Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:21:44.734863499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:21:44.738045962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:21:45.053139126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:21:45.055985381Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:21:47.476600478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:21:47.477678959Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-02T00:21:47.557843219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:21:47.563493604Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:21:50.810517571Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:21:51.568080877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:21:51.571355519Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:21:51.667715541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:21:51.671685239Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=warn ts=2026-05-02T00:21:52.476804746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:21:53.94514929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:21:53.948478164Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:21:54.144403154Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=debug ts=2026-05-02T00:21:57.457541306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:21:57.45757003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:21:57.47700686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:21:57.477028206Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:21:57.477036029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:21:57.477066622Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-02T00:21:57.477067837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:21:57.801267394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:21:57.804493915Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:22:00.810537869Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-02T00:22:00.969076626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:22:00.971961772Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:22:01.711816791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:22:01.714751715Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:22:02.06193306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:22:02.06563835Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:22:02.132731853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:22:02.135714632Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-02T00:22:02.477004433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:22:03.513542137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:22:04.144385314Z caller=checkpoint.go:372 msg="writing series" size="621 kB"
level=debug ts=2026-05-02T00:22:04.144407778Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=11
level=info ts=2026-05-02T00:22:04.161138736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:22:04.166134319Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:22:06.069567013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:22:06.072546423Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:22:07.476416703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:07.477686357Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-02T00:22:10.810490703Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-02T00:22:11.101996724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:22:11.107256725Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:22:11.184581144Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:22:12.476303049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:22:13.448777396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:22:13.452361824Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:22:14.144394816Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-02T00:22:16.692906863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:22:16.700072938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:22:17.476270234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:17.477462683Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-02T00:22:17.976832033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:22:20.618937765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:22:20.622651832Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=debug ts=2026-05-02T00:22:20.811563109Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-02T00:22:22.476720445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:22:22.643427435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:22:22.6502902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:22:22.650310547Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:22:22.650339078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:22:22.657258383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:22:24.143901184Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-02T00:22:24.143924593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-02T00:22:24.79484452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56402"
level=info ts=2026-05-02T00:22:26.623022225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:22:26.630454103Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:22:27.476171068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:22:27.47620088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:27.476208997Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:22:27.476219316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:22:27.47730208Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:22:29.494617095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:22:29.498396843Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:22:30.270481928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:22:30.273415978Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:22:30.810783661Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-02T00:22:31.501062839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:22:31.504676507Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:22:32.476350479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:22:33.515339195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:22:34.143827154Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-02T00:22:35.514370484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:22:35.521779604Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:22:35.767561772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:22:35.770993453Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:22:35.999448211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:22:36.002569908Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:22:36.205096563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:22:36.208193655Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:22:37.476975133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:37.477025096Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-02T00:22:38.156324834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:22:38.162526523Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:22:40.811206458Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-02T00:22:41.78453881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:22:41.789471371Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:22:42.476701198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:22:43.943880712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:22:43.947205323Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:22:44.143718336Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-02T00:22:45.511650892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:22:45.514389544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:22:46.093075855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:22:46.123062936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:22:46.125740247Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:22:47.176687541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:22:47.18056074Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=warn ts=2026-05-02T00:22:47.476649692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:47.477867653Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-02T00:22:50.411300599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:22:50.41670056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:22:50.811392566Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:22:50.81143312Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-02T00:22:52.476555532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:22:53.24421225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:22:53.248041792Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-02T00:22:54.144417001Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:22:55.146132002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:22:55.149569623Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:22:57.45715602Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:22:57.458210208Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:22:57.476588673Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:22:57.4766115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:22:57.476621966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:22:57.476615757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:22:57.477747802Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-02T00:22:58.662054463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:22:58.665404995Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:23:00.425186471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:23:00.429762641Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:23:00.811057595Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-02T00:23:01.012643678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:23:01.016345277Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=warn ts=2026-05-02T00:23:02.476649778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:02.650893644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:23:02.654125216Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
ts=2026-05-02T00:23:03.51623974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:23:04.144977076Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-02T00:23:05.535281135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:23:05.542558065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:23:07.476541293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:07.477627056Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:23:08.663841927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:23:08.667097248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:23:08.793360605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:23:08.800794266Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-02T00:23:10.811012386Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-02T00:23:12.47667019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:12.975073097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:23:12.978194944Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:23:14.144232259Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-02T00:23:17.258724839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:23:17.260312786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T00:23:17.477004315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:17.477164647Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-02T00:23:17.477180784Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-02T00:23:17.655808724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:23:17.662771288Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:23:18.692372727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:23:18.697957501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:23:19.0248626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:23:20.810898991Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=warn ts=2026-05-02T00:23:22.476678077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:22.784698444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:23:22.788622026Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:23:23.220245037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:23:23.223714375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:23:24.043159602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:23:24.046103295Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:23:24.144559174Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-02T00:23:25.747475916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:23:25.750869282Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:23:27.476421372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:27.476449671Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:23:27.476468733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:23:27.476477393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:23:27.500065017Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-02T00:23:27.50008834Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=3
level=info ts=2026-05-02T00:23:27.60695827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:23:27.609972676Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:23:30.810731233Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-02T00:23:31.972128853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:23:31.975897091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-02T00:23:32.47643684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:32.621827261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:23:32.62514984Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:23:32.799905013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:23:32.803753389Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
ts=2026-05-02T00:23:33.5176521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:23:34.165400736Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:23:34.165426783Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
level=info ts=2026-05-02T00:23:36.840783827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:23:36.844626697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-02T00:23:37.476363773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:37.477454215Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-02T00:23:40.810840517Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-02T00:23:40.86311841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:23:40.866045013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:23:41.905061202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:23:41.908783385Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:23:41.945983404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:23:41.94852613Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-02T00:23:42.476332013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:44.144380838Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=info ts=2026-05-02T00:23:45.157144801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:23:45.160406316Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=warn ts=2026-05-02T00:23:47.476568023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:47.479738602Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-02T00:23:47.479754763Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=4
level=debug ts=2026-05-02T00:23:47.626105472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:23:50.154199305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:23:50.157200555Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:23:50.816107831Z caller=checkpoint.go:372 msg="writing series" size="6.0 MB"
level=debug ts=2026-05-02T00:23:50.816131315Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.0 MB" series=1
level=info ts=2026-05-02T00:23:51.102356369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:23:51.105629487Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:23:51.471811658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:23:51.47597101Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:23:52.477218815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:23:52.588444528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:23:52.595686014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:23:52.595709636Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:23:52.595738212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:23:52.602306633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:23:54.144120638Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-02T00:23:54.220865515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:23:54.223550292Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:23:54.443922438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-02T00:23:57.456766648Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:23:57.457835757Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:23:57.476257048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:23:57.47626736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:23:57.476276243Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:23:57.476284855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:23:57.478154013Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-02T00:23:57.478164877Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-02T00:23:59.710341078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:23:59.713422191Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:24:00.811806443Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=info ts=2026-05-02T00:24:01.094283349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:24:01.100485037Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:24:02.476362853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:24:03.519006241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:24:04.14386875Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-02T00:24:06.085506746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:24:06.093576396Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:24:06.269048671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:24:06.272384618Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:24:06.272409603Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:24:06.272432769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:24:06.275175021Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-02T00:24:07.476374024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:07.477495935Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-02T00:24:07.898401962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:24:07.906134436Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:24:10.07928351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:24:10.082438346Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:24:10.817482096Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:24:10.817504734Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-05-02T00:24:11.500768484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:24:11.507845816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:24:12.476970313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:14.144629605Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=info ts=2026-05-02T00:24:16.051800644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:24:16.058273688Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:24:16.588677859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:24:16.592880884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:24:17.476712924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:17.477799031Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:24:18.348477273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:24:18.351412582Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:24:19.777123943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:24:19.780777496Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:24:20.810631241Z caller=checkpoint.go:372 msg="writing series" size="938 B"
ts=2026-05-02T00:24:21.081326731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40284"
level=info ts=2026-05-02T00:24:21.640269019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:24:21.64396087Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:24:21.740172188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:24:21.74387583Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:24:22.308364639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:24:22.312079407Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:24:22.476688624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:24:22.709383111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:24:22.712538188Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:24:22.723730968Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:24:24.144418366Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-02T00:24:24.144439771Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-02T00:24:25.567147979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:24:25.571389967Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:24:27.279922109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:24:27.283080658Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:24:27.477052506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:24:27.477081765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:24:27.477090713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:24:27.477117006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:27.477399016Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=info ts=2026-05-02T00:24:27.765069761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:24:27.768337773Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=debug ts=2026-05-02T00:24:30.810671108Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:24:32.476693556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:24:33.520685324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:24:34.145042792Z caller=checkpoint.go:372 msg="writing series" size="826 kB"
level=debug ts=2026-05-02T00:24:34.145063891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-02T00:24:34.29362237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:24:34.296681757Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-02T00:24:37.476502539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:37.477619848Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:24:38.097931126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:24:38.101358041Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:24:38.765505242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:24:38.773089469Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:24:39.953450026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:24:39.956990169Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:24:40.811139769Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-02T00:24:42.477019836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:44.144240528Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-02T00:24:45.294932604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:24:45.299086449Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:24:45.356291067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:24:45.359295557Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:24:45.815602033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:24:45.821841967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:24:47.476444076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:24:47.478682753Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:24:47.478698276Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-05-02T00:24:47.551748082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:24:47.554577193Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:24:48.627096711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:24:49.801595976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:24:49.80484534Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:24:50.810669796Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-02T00:24:52.476692419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:24:52.583999697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:24:52.587359521Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:24:53.626096753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:24:53.62910246Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:24:54.14456164Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-02T00:24:56.050036236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:24:57.457412742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:24:57.458448503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:24:57.476815831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-02T00:24:57.476810703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:24:57.476833554Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-02T00:24:57.476831408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:24:57.476841304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-05-02T00:24:57.486633725Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:24:57.54505335Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:24:57.545087199Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=debug ts=2026-05-02T00:25:00.811302675Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-02T00:25:01.820303794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:25:01.830298415Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-02T00:25:02.476594498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:25:03.521876599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:25:04.14698632Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-02T00:25:04.947960981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:25:04.958699434Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
ts=2026-05-02T00:25:05.28535008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57598"
level=info ts=2026-05-02T00:25:07.435474941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:25:07.453449001Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:25:07.47617738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:07.479066874Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-02T00:25:07.581308026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:25:07.58512224Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:25:10.811165799Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:25:10.811192413Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-02T00:25:11.57431371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:25:11.578565174Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:25:11.578589348Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:25:11.578611732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:25:11.581821913Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:25:12.476766209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:14.143846517Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:25:14.417495189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:25:14.421147947Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:25:15.268903595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:25:15.272373285Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:25:15.352578004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:25:15.355460815Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:25:16.566968807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:25:16.572502764Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:25:17.476189165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:17.477862787Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-02T00:25:18.579587217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:25:18.582451404Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:25:18.582470899Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:25:18.582491251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:25:18.584827111Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:25:19.130169231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:25:19.132914545Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:25:20.810862033Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=info ts=2026-05-02T00:25:21.004275049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:25:21.007167566Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:25:22.330305481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:25:22.333792876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:25:22.477002494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:22.867686462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:25:24.143871017Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-02T00:25:25.216767977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:25:25.220576764Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:25:27.476178875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:25:27.47617918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:27.476211732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:25:27.476221792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:25:27.47727421Z caller=checkpoint.go:384 msg="flushing series" totalSize="778 kB" series=4
level=info ts=2026-05-02T00:25:27.483126633Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-02T00:25:27.484686011Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007723919s
level=info ts=2026-05-02T00:25:28.153395197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:25:28.15704878Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:25:29.235970158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:25:29.239215342Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:25:31.30969782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:25:31.31279182Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:25:32.476419459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:25:33.525489041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=info ts=2026-05-02T00:25:33.824811728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:25:33.833320166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:25:34.931493317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:25:34.934981465Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:25:34.935004058Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:25:34.935040899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:25:34.937652824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:25:35.806720127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:25:35.813116555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:25:37.476561705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:25:39.705811349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:25:39.710050942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:25:42.476460916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:25:43.60922905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:25:43.61227385Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:25:44.704487936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:25:45.409647051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:25:45.417265202Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:25:46.523051921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:25:46.526013145Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:25:46.526035425Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:25:46.526056537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:25:46.528389727Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-02T00:25:47.476990191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:25:47.820426431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:25:47.824514175Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:25:48.473217454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:25:48.476097434Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:25:49.731600929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:25:49.734740859Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:25:50.579414645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:25:50.586686666Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:25:52.121767478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"build-e2e-nfdv\"}"
level=debug ts=2026-05-02T00:25:52.121822339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-02T00:25:52.125489683Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=warn ts=2026-05-02T00:25:52.476600149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:25:55.600809796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:25:55.604974887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:25:56.842085667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:25:56.844903077Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:25:57.456650967Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:25:57.457698829Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:25:57.476039287Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-02T00:25:57.476167728Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-02T00:25:57.476178478Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-02T00:25:57.476194939Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.878µs success=true
level=info ts=2026-05-02T00:25:57.476262481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:25:57.476285796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:25:57.476296737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:25:57.476362748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:25:57.47644758Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:25:58.785159498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:25:58.78998657Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:26:00.401160699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:26:00.403989482Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:26:00.598688952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:26:00.604168673Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=debug ts=2026-05-02T00:26:00.810787889Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-02T00:26:02.476475794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:26:03.52719467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=info ts=2026-05-02T00:26:03.564986415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:26:03.568318508Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=debug ts=2026-05-02T00:26:04.144275926Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-02T00:26:04.709607614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:26:04.712632503Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:26:04.712651366Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:04.712672729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:26:04.715043328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-02T00:26:05.183360192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:26:05.186294896Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:26:05.186320487Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:05.186342577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:26:05.188882154Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-02T00:26:05.2894507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46804"
level=info ts=2026-05-02T00:26:05.483808781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:26:05.486651574Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-02T00:26:07.476500204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:07.476523744Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-02T00:26:08.367873435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:26:08.370995015Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:26:10.539363629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:26:10.542835244Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:26:10.811134773Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-02T00:26:10.811159124Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-02T00:26:12.477124248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:14.143319003Z caller=checkpoint.go:372 msg="writing series" size="360 kB"
level=info ts=2026-05-02T00:26:16.184139323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:26:16.187656355Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-02T00:26:16.187679367Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:16.187702041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:26:16.190146471Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:26:16.39897958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:26:16.401747656Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:26:16.789910739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:26:16.794675353Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:26:17.476951072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:17.477229861Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=info ts=2026-05-02T00:26:19.148126349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:26:19.153617423Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:26:19.153638039Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:19.153664012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:26:19.158700747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:26:19.463814711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:26:19.467383247Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-02T00:26:20.810930109Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=debug ts=2026-05-02T00:26:20.810973216Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-02T00:26:20.980345736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:26:20.983320195Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:26:21.665559161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:26:21.672296516Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:26:22.476365748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:24.143122932Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:26:24.444006104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:26:24.447159106Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:26:24.779225215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:26:24.78335941Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-02T00:26:24.812382652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44886"
level=info ts=2026-05-02T00:26:25.735388409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:26:25.738923385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:26:26.398981125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:26:26.401674859Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-02T00:26:26.424930736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:26:27.476528328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:26:27.476555486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:26:27.476566235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:26:27.476563173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:27.477685123Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-02T00:26:27.47770302Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=info ts=2026-05-02T00:26:28.194455621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:26:28.197597495Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-02T00:26:28.268564795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:26:28.945654038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:26:28.953569837Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:26:30.230772831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:26:30.236723316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:26:30.810472653Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-02T00:26:32.476960117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:26:33.528638285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=info ts=2026-05-02T00:26:33.645967895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:26:33.652663775Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:26:33.822957496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:26:33.825978265Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:26:33.825997035Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:33.826016301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:26:33.828358708Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:26:34.143405024Z caller=checkpoint.go:372 msg="writing series" size="822 B"
level=info ts=2026-05-02T00:26:36.468994608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:26:36.471879047Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=debug ts=2026-05-02T00:26:37.476883114Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-02T00:26:37.476891531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:26:37.892202318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:26:37.896212945Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:26:39.100126115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:26:39.104176322Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:26:39.104199691Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:39.104227355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:26:39.10819038Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:26:39.154409977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:26:39.161849234Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=error ts=2026-05-02T00:26:39.161868329Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:39.40446449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:26:39.412281748Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:26:40.810530791Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-02T00:26:42.476337128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:44.144123398Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-02T00:26:45.675360849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:26:45.678407351Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:26:47.476805926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:47.47684977Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-02T00:26:48.180577904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:26:48.183858221Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:26:48.721374979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:26:48.724055858Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:26:48.724074868Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:48.72409599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:26:48.726470643Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:26:48.831719838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:26:48.834473592Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:26:49.277863678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:26:49.281004429Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:26:49.281025663Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:49.281047603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:26:49.283715832Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=debug ts=2026-05-02T00:26:50.477118458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:26:50.573283618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:26:50.576282577Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:26:50.576304737Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:50.576327392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:26:50.578822368Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:26:50.810417749Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:26:51.838104356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:26:51.841465857Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-02T00:26:52.476389329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:26:53.625610541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:26:53.628616284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:26:53.628637876Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:26:53.628660628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:26:53.631252081Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:26:54.144197207Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
ts=2026-05-02T00:26:54.814854012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56886"
level=debug ts=2026-05-02T00:26:57.45732962Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:26:57.458375977Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:26:57.473667442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:26:57.476197847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:26:57.476213935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:26:57.476221429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:26:57.476228238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:26:57.476451377Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=error ts=2026-05-02T00:26:57.47754341Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:26:58.070134662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:26:58.073086828Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:26:58.362779425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:26:58.369505479Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:26:59.478698051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:26:59.482876076Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:27:00.810340607Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-02T00:27:01.639634503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:27:01.642963988Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:27:01.642993586Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:27:01.643021264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:27:01.645420377Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:27:02.148991916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:27:02.155782788Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:27:02.476543197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:02.793698566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:27:02.796795668Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:27:03.437867841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:27:03.441657927Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-02T00:27:03.530593978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:27:04.143610978Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-02T00:27:04.143635278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=11
level=info ts=2026-05-02T00:27:04.501289812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:27:04.504788008Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:27:05.921862063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:27:05.926227434Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-02T00:27:07.476482278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:27:07.476781706Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=info ts=2026-05-02T00:27:10.671843633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:27:10.675052447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-02T00:27:10.810369685Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-02T00:27:11.424883828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:27:11.427608153Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-02T00:27:12.044339446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:27:12.0478508Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:27:12.04787247Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:27:12.047897205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:27:12.051019118Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-02T00:27:12.476324766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:13.629996621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:27:13.633079346Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-02T00:27:13.684377775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:27:13.68719499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:27:13.687209214Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:27:13.687228335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:27:13.689599913Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:27:13.989352129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:27:13.992571846Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:27:14.14412785Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-02T00:27:17.476615567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:27:17.47665747Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-02T00:27:19.723740684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="146 MB" avg_uncomp="21 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:27:19.733821768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-02T00:27:20.809995403Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-02T00:27:21.091020415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45498"
level=info ts=2026-05-02T00:27:21.109430772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:27:21.117724555Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:27:21.420490326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:27:21.423538153Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:27:21.511977957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:27:21.514807781Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-02T00:27:21.98121242Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:27:22.47717864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:23.927398482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:27:23.931325466Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-02T00:27:24.144138781Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-02T00:27:24.144161275Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-02T00:27:24.817345986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60020"
level=info ts=2026-05-02T00:27:25.290316177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:27:25.298493715Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=info ts=2026-05-02T00:27:26.2602694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:27:26.265127568Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:27:26.379387395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:27:26.382813119Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:27:27.13395763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:27:27.138090473Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:27:27.476491574Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:27:27.476515271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:27:27.476523111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:27:27.476529262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:27:27.476564529Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-02T00:27:28.929599092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:27:28.932745181Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=debug ts=2026-05-02T00:27:29.70744861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:27:29.709061006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-02T00:27:30.810168881Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-02T00:27:31.698359855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:27:31.701424488Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:27:32.477124682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:27:33.532500457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:27:34.143292337Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-02T00:27:34.654892227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:27:34.658051385Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:27:37.075499748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:27:37.078593133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-02T00:27:37.476435656Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-02T00:27:37.476468916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:40.277375466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:27:40.280227588Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:27:40.719511477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:27:40.722519882Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:27:40.793743469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:27:40.796675491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:27:40.810730577Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-02T00:27:42.43191503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:27:42.437683839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-02T00:27:42.476446202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:42.598750411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:27:42.601905458Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:27:43.499451505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:27:43.502935876Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:27:43.502976407Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:27:43.503003693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:27:43.506011771Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:27:44.14316929Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-02T00:27:45.436606016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:27:45.440044255Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:27:47.476577212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:27:47.476706021Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-02T00:27:49.994695796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:27:50.810690243Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-02T00:27:50.810712323Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
ts=2026-05-02T00:27:51.092552434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54534"
level=warn ts=2026-05-02T00:27:52.476442974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:53.164059875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:27:53.169303028Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:27:54.14408157Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-02T00:27:54.634168947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:27:54.637372444Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:27:57.282417981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:27:57.286115235Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-02T00:27:57.456971802Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:27:57.458039257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:27:57.47649265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:27:57.476514245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-02T00:27:57.476508704Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-02T00:27:57.476522821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:27:57.476528817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:27:57.970431669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:27:57.975006674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:27:57.975026999Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:27:57.975048553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:27:57.978384705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:28:00.686597294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:28:00.690822341Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:28:00.810170676Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-02T00:28:01.358962798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="169 MB" avg_uncomp="21 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:28:01.36550533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:28:02.218994592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:28:02.222288743Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:28:02.476964182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:28:02.722312245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:28:02.725002958Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-02T00:28:03.533559118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:28:04.144219353Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-02T00:28:05.170233378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:28:05.17732767Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-02T00:28:07.476809941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:07.476892532Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-02T00:28:07.837165599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:28:07.840211992Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:28:09.665105573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:28:09.668204249Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=debug ts=2026-05-02T00:28:10.81041794Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-05-02T00:28:12.476871689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:28:12.636872385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:28:12.640904291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:28:12.698875905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:28:12.701792897Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:28:13.48867813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:28:13.496515184Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:28:14.143347043Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-02T00:28:16.045749033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:28:16.048777859Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:28:16.916859031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:28:17.132387649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:28:17.135719907Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-02T00:28:17.476858074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:17.47702754Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-02T00:28:17.477043398Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-02T00:28:20.810382239Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-02T00:28:20.97390572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:28:20.977897179Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
ts=2026-05-02T00:28:21.094150125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50430"
level=warn ts=2026-05-02T00:28:22.476911304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:28:23.893132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:28:23.899395164Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:28:24.144232418Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-02T00:28:24.361775509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:28:24.367568109Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:28:25.57735629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:28:25.58007562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-02T00:28:25.920647922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:28:25.923975301Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:28:26.118491626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:28:26.121901176Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-02T00:28:27.092638179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:28:27.477056452Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-02T00:28:27.477073011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:27.477083268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:28:27.477092086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-02T00:28:27.498227027Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-02T00:28:27.49824655Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=3
level=info ts=2026-05-02T00:28:28.609464737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:28:28.612496671Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:28:30.170784874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:28:30.174891024Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=debug ts=2026-05-02T00:28:30.810624036Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-02T00:28:32.476836077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:28:33.385117192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:28:33.388708927Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-02T00:28:33.534377961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:28:34.164038814Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-02T00:28:34.164065272Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=2
level=info ts=2026-05-02T00:28:35.123476265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:28:35.126240974Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-02T00:28:37.476204031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:37.477329785Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-02T00:28:39.68076871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:28:39.684230154Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-02T00:28:39.981057405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:28:39.985211524Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-02T00:28:40.810693729Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-02T00:28:42.476370767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:44.144299598Z caller=checkpoint.go:372 msg="writing series" size="719 kB"
level=info ts=2026-05-02T00:28:46.847973726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.5 MB" total_uncomp="140 MB" avg_uncomp="16 MB" full=7 synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:28:46.855832148Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:28:46.85585588Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-02T00:28:46.85588389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.5 MB" total_uncomp="140 MB" avg_uncomp="16 MB" full=7 synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:28:46.863201952Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:28:47.350784194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:28:47.354009155Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-02T00:28:47.476165624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:47.480219033Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-02T00:28:47.480235317Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=4
level=debug ts=2026-05-02T00:28:48.134819186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-02T00:28:48.214990598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="169 MB" avg_uncomp="21 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:28:48.222309972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-02T00:28:49.037703159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:28:49.041757292Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:28:50.817026684Z caller=checkpoint.go:372 msg="writing series" size="6.1 MB"
level=debug ts=2026-05-02T00:28:50.817054212Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.1 MB" series=1
level=warn ts=2026-05-02T00:28:52.476370423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:28:53.03665587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:28:53.039504869Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:28:54.143695113Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-02T00:28:57.002290247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:28:57.005389226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-02T00:28:57.112707732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:28:57.12022975Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=debug ts=2026-05-02T00:28:57.457425691Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:28:57.458489326Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-02T00:28:57.476846722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:28:57.476861907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:28:57.476869663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:28:57.476912243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:28:57.478580202Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-02T00:28:57.478591222Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-05-02T00:28:58.160894187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:28:58.163845182Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-02T00:28:58.405460278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:28:58.408500621Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-02T00:29:00.810621522Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=info ts=2026-05-02T00:29:01.679822016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:29:01.683588504Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=warn ts=2026-05-02T00:29:02.476296393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:29:03.328000904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.5 MB" total_uncomp="140 MB" avg_uncomp="16 MB" full=7 synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:29:03.335278046Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-02T00:29:03.535225146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:29:04.143752682Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-02T00:29:05.954660187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:29:05.958299849Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:29:06.176496035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:29:06.17957071Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-02T00:29:07.476922446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:07.477002644Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-02T00:29:08.284235843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:29:08.287649029Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=info ts=2026-05-02T00:29:09.663642566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:29:09.670820416Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-02T00:29:10.817174119Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-02T00:29:10.81719809Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-02T00:29:12.476832288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:29:13.296415848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:29:13.300740311Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:29:13.888504692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:29:13.891571822Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:29:14.14419084Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=debug ts=2026-05-02T00:29:16.309035335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:29:16.81034679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:29:16.814028288Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:29:17.477007165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:17.477052349Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-02T00:29:18.799645279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:29:18.802986142Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:29:19.817572829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:29:19.820516908Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-02T00:29:19.87397652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:29:20.810368672Z caller=checkpoint.go:372 msg="writing series" size="938 B"
level=warn ts=2026-05-02T00:29:22.476991625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:29:23.722534261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:29:23.72674546Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-02T00:29:24.144341695Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-02T00:29:24.144367004Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-02T00:29:24.826411084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60456"
level=info ts=2026-05-02T00:29:25.54761028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:29:25.550781112Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:29:26.942530177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-02T00:29:26.945561834Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19de5b8cb28%3A19de5b8d72a%3Af5e3efef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-02T00:29:26.993634535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:29:26.996331702Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=info ts=2026-05-02T00:29:27.476699705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:29:27.476738165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:29:27.476746814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:29:27.476757729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:27.477130901Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=info ts=2026-05-02T00:29:27.796504012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:29:27.799793139Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:29:30.810091716Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-02T00:29:32.476996444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:29:33.53619154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:29:34.144201349Z caller=checkpoint.go:372 msg="writing series" size="838 kB"
level=debug ts=2026-05-02T00:29:34.144224581Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-02T00:29:34.889855767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:29:34.894341427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-02T00:29:37.120676999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-02T00:29:37.12384232Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19de5b8cb28%3A19de5b94ca8%3Ac7bd652d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-02T00:29:37.476190523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:37.477305991Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-02T00:29:37.819076108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.5 MB" total_uncomp="140 MB" avg_uncomp="16 MB" full=7 synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-02T00:29:37.827196049Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19de5b85ec5%3A19de5d98850%3A670ddffb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-02T00:29:38.436224697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="59 MB" avg_comp="2.7 MB" total_uncomp="487 MB" avg_uncomp="22 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-02T00:29:38.440278962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19de5b859e5%3A19de5be9c08%3A94e5b612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-02T00:29:40.528339463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-02T00:29:40.531990957Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19de5b8cb51%3A19de5b8d72a%3Ab5b70201?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-02T00:29:40.809774051Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-02T00:29:40.881139352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="169 MB" avg_uncomp="21 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-02T00:29:40.887926892Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19de5b851f6%3A19de5b8d70a%3A486044a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-02T00:29:42.476656929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:44.143716225Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-02T00:29:47.008805468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fdd27b277198e14c immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-svca\"}"
level=error ts=2026-05-02T00:29:47.021296136Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fdd27b277198e14c/19de5c92a90%3A19de5ddef14%3A6b1d0c4c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-svca\"}"
level=warn ts=2026-05-02T00:29:47.47699015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:47.478036727Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-02T00:29:47.478054847Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-05-02T00:29:48.535492345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-02T00:29:48.538334677Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19de5b85c61%3A19de5b8d72a%3A5812578e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-02T00:29:49.853115849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-02T00:29:49.856393895Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19de5c819b6%3A19de5c96cf3%3A1701d3ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-02T00:29:50.156336523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-02T00:29:50.810101415Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-02T00:29:52.138784227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-02T00:29:52.142010373Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19de5b85c84%3A19de5b8fa03%3Aeaa57901?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-02T00:29:52.477187233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:54.144010808Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-02T00:29:55.714324919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-02T00:29:55.718068589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19de5b85102%3A19de5d35421%3A30071275?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-02T00:29:56.655901613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3a47474f3ef0eea immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nfdv\"}"
level=error ts=2026-05-02T00:29:56.659050176Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3a47474f3ef0eea/19de5c8ff99%3A19de5dcd268%3A18bdd3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nfdv\"}"
level=info ts=2026-05-02T00:29:57.125192966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=3 total_comp="5.9 MB" avg_comp="2.0 MB" total_uncomp="34 MB" avg_uncomp="11 MB" full=1 synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-02T00:29:57.131585171Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19de5b84ddf%3A19de5d2baba%3A10badb05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-02T00:29:57.310054526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=569217daa661b643 immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-vbuj\"}"
level=error ts=2026-05-02T00:29:57.314125491Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/569217daa661b643/19de5c94db9%3A19de5d75bf8%3A3ab3fb48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-vbuj\"}"
level=info ts=2026-05-02T00:29:57.457577479Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-02T00:29:57.457604333Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-02T00:29:57.477039662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-02T00:29:57.47706117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-02T00:29:57.477072625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-02T00:29:57.477079001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:29:57.550014147Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-02T00:29:57.550048437Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=debug ts=2026-05-02T00:29:57.739510478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T00:30:00.468798902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-02T00:30:00.477849273Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19de5b8cb51%3A19de5b8cb52%3A46ec5409?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-02T00:30:00.81082156Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-02T00:30:02.477313837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-02T00:30:03.538184106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-33c59d6d 10.128.0.86:7946"
level=debug ts=2026-05-02T00:30:04.144330657Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-02T00:30:05.302467736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57630"
level=info ts=2026-05-02T00:30:05.622241875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-02T00:30:05.641200602Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19de5b85e39%3A19de60d1db0%3Af84b20cc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-02T00:30:06.107586613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-02T00:30:06.118074339Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19de5b8508b%3A19de5b8cb2a%3A867f6983?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-02T00:30:06.119067899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-02T00:30:06.129508457Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19de5b84ddf%3A19de605075f%3A2d3d3433?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-02T00:30:07.476177233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:30:07.4783134Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-02T00:30:08.481790914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-02T00:30:08.485251242Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19de5b8bd19%3A19de5d626c3%3Afb2824f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-02T00:30:10.811407119Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-05-02T00:30:10.81143123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-02T00:30:12.455460084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5fb7e19fa07b2fc7 immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"build-e2e-lsui\"}"
level=error ts=2026-05-02T00:30:12.458729652Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5fb7e19fa07b2fc7/19de5c824d8%3A19de5ca18d4%3A8b4d8751?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lsui\"}"
level=warn ts=2026-05-02T00:30:12.476982597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-02T00:30:12.525498563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=364c2d36e1fa8fc2 immediate=false num_chunks=1 total_comp="208 kB" avg_comp="208 kB" total_uncomp="917 kB" avg_uncomp="917 kB" full=1 labels="{stream=\"integration2-txvq\"}"
level=error ts=2026-05-02T00:30:12.528657483Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/364c2d36e1fa8fc2/19de5d3ba47%3A19de5d7ae01%3A9e31ea36?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-txvq\"}"
level=info ts=2026-05-02T00:30:12.781791793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-02T00:30:12.784970967Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19de5b8bd19%3A19de5b8d72a%3A341e9a5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-02T00:30:14.143312908Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-02T00:30:14.208454304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-02T00:30:14.212118345Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19de5b8bd19%3A19de5b909a9%3Ac86fae75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-02T00:30:15.116864367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-02T00:30:15.120827506Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19de5b84ddf%3A19de5fe5bbb%3Ab9a27b80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-02T00:30:16.934679763Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-02T00:30:17.47654749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-02T00:30:17.477324209Z caller=checkpoint.go:372 msg="writing series" size="696 kB"